Merge pull request #1187

Remove port type
tenzir · Nov 23, 2020 · 722c9da · 722c9da
2 parents 43661f0 + 08bba5e
commit 722c9da
Show file tree

Hide file tree

Showing 97 changed files with 21,457 additions and 21,706 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,10 +12,20 @@ Every entry has a category for which we use the following visual abbreviations:
 
 ## Unreleased
 
+- ⚡️ The `port` type is no longer a first-class type. The new way to represent
+  transport-layer ports relies on `count` instead. In the schema, VAST ships with
+  a new alias `type port = count` to keep existing schema definitions in tact.
+  However, this is a breaking change because the on-disk format and Arrow data
+  representation changed. Queries with `:port` type extractors no longer work.
+  Similarly, the syntax `53/udp` no longer exists; use `count` syntax `53`
+  instead. Since most `port` occurrences do not carry a known transport-layer
+  type, and the type information exists typically in a separate field, removing
+  `port` as native type streamlines the data model.
+
 - 🐞 VAST no longer blocks when an invalid query operation is issued.
   [#1189](https://github.com/tenzir/vast/pull/1189)
 
-- 🎁 The storage required for index ip addresses has been optimized. This
+- 🎁 The storage required for index IP addresses has been optimized. This
   should result in significantly reduced memory usage over time, as well
   as faster restart times and reduced disk space requirements.
   [#1172](https://github.com/tenzir/vast/pull/1172)

diff --git a/integration/data/json/string-types.json b/integration/data/json/string-types.json
@@ -1,2 +1,2 @@
 
-{"a": "foo", "b": "1.2.3.4", "c": "10.0.0.0/8", "d": "53/udp", "e": "now", "f": "2017-08-13+21:10:42", "g": "2017-08-13", "h": "42hours", "i": "-1 min"}
+{"a": "foo", "b": "1.2.3.4", "c": "10.0.0.0/8", "d": "2 secs", "e": "now", "f": "2017-08-13+21:10:42", "g": "2017-08-13", "h": "42hours", "i": "-1 min"}
diff --git a/integration/default_set.yaml b/integration/default_set.yaml
@@ -97,16 +97,16 @@ tests:
         input: data/zeek/conn.log.gz
       - command: -N count ":addr == 192.168.1.104"
       - command: -N count -e ":addr == 192.168.1.104"
-      - command: -N count "resp_p == 80/tcp"
-      - command: -N count "resp_p != 80/tcp"
+      - command: -N count "resp_p == 80"
+      - command: -N count "resp_p != 80"
   Node Zeek conn log:
     tags: [node, import-export, zeek]
     steps:
       - command: -N import zeek
         input: data/zeek/conn.log.gz
       - command: -N export ascii 'resp_h == 192.168.1.104'
       - command: -N export ascii 'orig_bytes > 1k && orig_bytes < 1Ki'
-      - command: -N export ascii ':string == "OrfTtuI5G4e" || :port == 67/udp'
+      - command: -N export ascii ':string == "OrfTtuI5G4e" || :count == 67' # TODO: make this :port == 67 once we support built-in type aliases
       - command: -N export ascii '#type == "zeek.conn" && resp_h == 192.168.1.104'
       - command: '-N export ascii "#type != \"zeek.conn\" && #type != \"vast.metrics\""'
       - command: -N export ascii '#type != "foobar" && resp_h == 192.168.1.104'
@@ -258,7 +258,7 @@ tests:
       - command: import -b pcap
         input: data/pcap/example.pcap.gz
       - command: export ascii ':addr in 192.168.168.0/24'
-      - command: export pcap 'sport > 60000/tcp && src !in 192.168.168.0/8'
+      - command: export pcap 'sport > 60000 && src !in 192.168.168.0/8'
 
   Manual 1 node:
     condition: version | jq -e '."PCAP"'
@@ -267,7 +267,7 @@ tests:
       - command: -N import pcap
         input: data/pcap/example.pcap.gz
       - command: -N export ascii ':addr in 192.168.168.0/24'
-      - command: -N export pcap 'sport > 60000/tcp && src !in 192.168.168.0/8'
+      - command: -N export pcap 'sport > 60000 && src !in 192.168.168.0/8'
       - command: -N status --detailed
         transformation: jq '.index.statistics'
 

diff --git a/integration/misc/schema/zeek-conn.schema b/integration/misc/schema/zeek-conn.schema
@@ -1,3 +1,5 @@
+type port = count
+
 type zeek.conn.custom = record{
   ts: time #timestamp,
   uid: string,

diff --git a/integration/reference/arrow/step_01.ref b/integration/reference/arrow/step_01.ref
@@ -6,9 +6,9 @@ done with current reader, rows: 10
 duration: int64
 history: string
 id.orig_h: fixed_size_binary[16]
-id.orig_p: fixed_size_binary[3]
+id.orig_p: uint64
 id.resp_h: fixed_size_binary[16]
-id.resp_p: fixed_size_binary[3]
+id.resp_p: uint64
 local_orig: bool
 missed_bytes: uint64
 name: 'zeek.conn'

diff --git a/integration/reference/example-config-file/step_01.ref b/integration/reference/example-config-file/step_01.ref
@@ -1,23 +1,23 @@
-<2009-11-18T16:59:33.941668096, "QqEBsEl5zOf", 192.168.1.103, 1397/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 2.65s, 2943, 1440, "SF", nil, 0, "ShAdDafF", 19, 3711, 30, 2648, []>
-<2009-11-18T17:23:50.304653056, "E8oxyJ4sSf4", 192.168.1.103, 1749/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 115.93ms, 5679, 1405, "SF", nil, 0, "ShAdDafF", 19, 6447, 30, 2613, []>
-<2009-11-18T17:26:14.919133952, "tkT6ftxDPmg", 192.168.1.103, 1755/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 118.43ms, 2357, 1405, "SF", nil, 0, "ShAdDafF", 17, 3045, 28, 2533, []>
-<2009-11-18T17:37:00.302089984, "PK1psr2Dg68", 192.168.1.105, 49218/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 1.6s, 4899, 1519, "SF", nil, 0, "ShAadDfF", 18, 5631, 29, 2691, []>
-<2009-11-18T17:38:00.189256960, "U6EoRx2yUyl", 192.168.1.105, 49219/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 22.98ms, 4429, 799, "SF", nil, 0, "ShAadDfF", 18, 5161, 30, 2011, []>
-<2009-11-18T17:40:50.124592128, "3cN64Zhy0Cf", 192.168.1.105, 49220/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 114.18ms, 2750, 1466, "SF", nil, 0, "ShAadDfF", 16, 3402, 27, 2558, []>
-<2009-11-18T17:42:03.706407936, "zkdnQLD4rx7", 192.168.1.103, 1806/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 33.24ms, 4371, 1370, "SF", nil, 0, "ShAdDafF", 17, 5059, 28, 2498, []>
-<2009-11-18T17:58:29.383510016, "6uaqOCphs9g", 192.168.1.102, 1400/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 4.19s, 1522, 1370, "SF", nil, 0, "ShAdDafF", 16, 2170, 26, 2418, []>
-<2009-11-18T18:00:48.261595904, "oAiWgG5cMng", 192.168.1.102, 1404/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 113.17ms, 5247, 1370, "SF", nil, 0, "ShAdDafF", 18, 5975, 28, 2498, []>
-<2009-11-18T18:01:29.262108928, "GxYIFhoLgza", 192.168.1.102, 1405/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 21.48ms, 5796, 632, "SF", nil, 0, "ShAdDafF", 19, 6564, 28, 1760, []>
-<2009-11-18T18:09:17.457758976, "3c03Zu0ccbh", 192.168.1.105, 49336/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 115.17ms, 2430, 1466, "SF", nil, 0, "ShAadDfF", 16, 3082, 27, 2558, []>
-<2009-11-18T18:13:56.508357888, "lhnlnKqXCxf", 192.168.1.105, 49353/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 111.93ms, 2462, 1466, "SF", nil, 0, "ShAadDfF", 16, 3114, 27, 2558, []>
-<2009-11-18T18:14:19.128662016, "ufFpSyrwjwd", 192.168.1.103, 1836/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 31.99ms, 3372, 1370, "SF", nil, 0, "ShAdDafF", 16, 4020, 26, 2418, []>
-<2009-11-18T18:24:27.549041152, "mpHi6R9SU9i", 192.168.1.102, 1709/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 113.92ms, 2374, 1370, "SF", nil, 0, "ShAdDafF", 16, 3022, 26, 2418, []>
-<2009-11-18T18:25:38.108254976, "Rdz0JuTWH7h", 192.168.1.105, 49561/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 32.98ms, 4334, 1466, "SF", nil, 0, "ShAadDfF", 17, 5026, 28, 2598, []>
-<2009-11-18T19:55:41.027461888, "K1ETvrXVgr", 192.168.1.104, 1572/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 438.74ms, 1021, 308, "SF", nil, 0, "ShAdDafF", 10, 1429, 17, 996, []>
-<2009-11-18T20:51:24.692600064, "4JRIv0bBLY5", 192.168.1.104, 1604/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 279.07ms, 761, 308, "SF", nil, 0, "ShAdDafF", 10, 1169, 17, 996, []>
-<2009-11-18T20:57:20.949762048, "iZiq8ubpoz5", 192.168.1.104, 1665/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 254.84ms, 574, 308, "SF", nil, 0, "ShAdDafF", 10, 982, 17, 996, []>
-<2009-11-18T22:02:37.587842816, "9ZNqUeBHGZg", 192.168.1.103, 1934/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 770.76ms, 5732, 1370, "SF", nil, 0, "ShAdDafF", 18, 6460, 27, 2458, []>
-<2009-11-18T22:08:27.588229888, "PbX5YfEjOJg", 192.168.1.103, 2008/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 1.23s, 3460, 1370, "SF", nil, 0, "ShAdDafF", 17, 4148, 26, 2418, []>
-<2009-11-18T22:33:34.298059008, "IcCYHXxqpC5", 192.168.1.102, 1911/tcp, 192.168.1.1, 25/tcp, "tcp", "smtp", 1.03s, 7079, 1370, "SF", nil, 0, "ShAdDafF", 20, 7887, 29, 2538, []>
-<2009-11-19T01:28:01.990388992, "4nUkGQ2lYSi", 192.168.1.105, 49307/tcp, 143.166.11.10, 64053/tcp, "tcp", "ftp-data", 41.39s, 0, 4255056, "RSTO", nil, 178020, "ShAdfR", 1453, 63572, 2965, 4198407, []>
-<2009-11-19T01:29:23.408284928, "EQbLqHeggqj", 192.168.1.105, 49330/tcp, 143.166.11.10, 64334/tcp, "tcp", "ftp-data", 27.61s, 0, 4255056, "RSTO", nil, 0, "ShAdfR", 1514, 67868, 3101, 4392906, []>
+<2009-11-18T16:59:33.941668096, "QqEBsEl5zOf", 192.168.1.103, 1397, 192.168.1.1, 25, "tcp", "smtp", 2.65s, 2943, 1440, "SF", nil, 0, "ShAdDafF", 19, 3711, 30, 2648, []>
+<2009-11-18T17:23:50.304653056, "E8oxyJ4sSf4", 192.168.1.103, 1749, 192.168.1.1, 25, "tcp", "smtp", 115.93ms, 5679, 1405, "SF", nil, 0, "ShAdDafF", 19, 6447, 30, 2613, []>
+<2009-11-18T17:26:14.919133952, "tkT6ftxDPmg", 192.168.1.103, 1755, 192.168.1.1, 25, "tcp", "smtp", 118.43ms, 2357, 1405, "SF", nil, 0, "ShAdDafF", 17, 3045, 28, 2533, []>
+<2009-11-18T17:37:00.302089984, "PK1psr2Dg68", 192.168.1.105, 49218, 192.168.1.1, 25, "tcp", "smtp", 1.6s, 4899, 1519, "SF", nil, 0, "ShAadDfF", 18, 5631, 29, 2691, []>
+<2009-11-18T17:38:00.189256960, "U6EoRx2yUyl", 192.168.1.105, 49219, 192.168.1.1, 25, "tcp", "smtp", 22.98ms, 4429, 799, "SF", nil, 0, "ShAadDfF", 18, 5161, 30, 2011, []>
+<2009-11-18T17:40:50.124592128, "3cN64Zhy0Cf", 192.168.1.105, 49220, 192.168.1.1, 25, "tcp", "smtp", 114.18ms, 2750, 1466, "SF", nil, 0, "ShAadDfF", 16, 3402, 27, 2558, []>
+<2009-11-18T17:42:03.706407936, "zkdnQLD4rx7", 192.168.1.103, 1806, 192.168.1.1, 25, "tcp", "smtp", 33.24ms, 4371, 1370, "SF", nil, 0, "ShAdDafF", 17, 5059, 28, 2498, []>
+<2009-11-18T17:58:29.383510016, "6uaqOCphs9g", 192.168.1.102, 1400, 192.168.1.1, 25, "tcp", "smtp", 4.19s, 1522, 1370, "SF", nil, 0, "ShAdDafF", 16, 2170, 26, 2418, []>
+<2009-11-18T18:00:48.261595904, "oAiWgG5cMng", 192.168.1.102, 1404, 192.168.1.1, 25, "tcp", "smtp", 113.17ms, 5247, 1370, "SF", nil, 0, "ShAdDafF", 18, 5975, 28, 2498, []>
+<2009-11-18T18:01:29.262108928, "GxYIFhoLgza", 192.168.1.102, 1405, 192.168.1.1, 25, "tcp", "smtp", 21.48ms, 5796, 632, "SF", nil, 0, "ShAdDafF", 19, 6564, 28, 1760, []>
+<2009-11-18T18:09:17.457758976, "3c03Zu0ccbh", 192.168.1.105, 49336, 192.168.1.1, 25, "tcp", "smtp", 115.17ms, 2430, 1466, "SF", nil, 0, "ShAadDfF", 16, 3082, 27, 2558, []>
+<2009-11-18T18:13:56.508357888, "lhnlnKqXCxf", 192.168.1.105, 49353, 192.168.1.1, 25, "tcp", "smtp", 111.93ms, 2462, 1466, "SF", nil, 0, "ShAadDfF", 16, 3114, 27, 2558, []>
+<2009-11-18T18:14:19.128662016, "ufFpSyrwjwd", 192.168.1.103, 1836, 192.168.1.1, 25, "tcp", "smtp", 31.99ms, 3372, 1370, "SF", nil, 0, "ShAdDafF", 16, 4020, 26, 2418, []>
+<2009-11-18T18:24:27.549041152, "mpHi6R9SU9i", 192.168.1.102, 1709, 192.168.1.1, 25, "tcp", "smtp", 113.92ms, 2374, 1370, "SF", nil, 0, "ShAdDafF", 16, 3022, 26, 2418, []>
+<2009-11-18T18:25:38.108254976, "Rdz0JuTWH7h", 192.168.1.105, 49561, 192.168.1.1, 25, "tcp", "smtp", 32.98ms, 4334, 1466, "SF", nil, 0, "ShAadDfF", 17, 5026, 28, 2598, []>
+<2009-11-18T19:55:41.027461888, "K1ETvrXVgr", 192.168.1.104, 1572, 192.168.1.1, 25, "tcp", "smtp", 438.74ms, 1021, 308, "SF", nil, 0, "ShAdDafF", 10, 1429, 17, 996, []>
+<2009-11-18T20:51:24.692600064, "4JRIv0bBLY5", 192.168.1.104, 1604, 192.168.1.1, 25, "tcp", "smtp", 279.07ms, 761, 308, "SF", nil, 0, "ShAdDafF", 10, 1169, 17, 996, []>
+<2009-11-18T20:57:20.949762048, "iZiq8ubpoz5", 192.168.1.104, 1665, 192.168.1.1, 25, "tcp", "smtp", 254.84ms, 574, 308, "SF", nil, 0, "ShAdDafF", 10, 982, 17, 996, []>
+<2009-11-18T22:02:37.587842816, "9ZNqUeBHGZg", 192.168.1.103, 1934, 192.168.1.1, 25, "tcp", "smtp", 770.76ms, 5732, 1370, "SF", nil, 0, "ShAdDafF", 18, 6460, 27, 2458, []>
+<2009-11-18T22:08:27.588229888, "PbX5YfEjOJg", 192.168.1.103, 2008, 192.168.1.1, 25, "tcp", "smtp", 1.23s, 3460, 1370, "SF", nil, 0, "ShAdDafF", 17, 4148, 26, 2418, []>
+<2009-11-18T22:33:34.298059008, "IcCYHXxqpC5", 192.168.1.102, 1911, 192.168.1.1, 25, "tcp", "smtp", 1.03s, 7079, 1370, "SF", nil, 0, "ShAdDafF", 20, 7887, 29, 2538, []>
+<2009-11-19T01:28:01.990388992, "4nUkGQ2lYSi", 192.168.1.105, 49307, 143.166.11.10, 64053, "tcp", "ftp-data", 41.39s, 0, 4255056, "RSTO", nil, 178020, "ShAdfR", 1453, 63572, 2965, 4198407, []>
+<2009-11-19T01:29:23.408284928, "EQbLqHeggqj", 192.168.1.105, 49330, 143.166.11.10, 64334, "tcp", "ftp-data", 27.61s, 0, 4255056, "RSTO", nil, 0, "ShAdfR", 1514, 67868, 3101, 4392906, []>