From b015fdfa263b75072e76e39d9a64293f4f58e087 Mon Sep 17 00:00:00 2001 From: Shani Date: Mon, 13 Nov 2023 17:12:37 +0200 Subject: [PATCH 1/2] Add Parquet output Issue #3682 https://github.com/aquasecurity/tracee/issues/3682 --- go.mod | 7 ++ go.sum | 36 ++++++ pkg/cmd/cobra/config.go | 2 + pkg/cmd/flags/output.go | 2 +- pkg/cmd/printer/printer.go | 42 +++++++ pkg/cmd/printer/printer_parquet.go | 178 +++++++++++++++++++++++++++++ 6 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 pkg/cmd/printer/printer_parquet.go diff --git a/go.mod b/go.mod index 0005ac3cafcd..e369ece6c8bb 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,8 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 // indirect + github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect + github.com/apache/thrift v0.14.2 // indirect github.com/containerd/typeurl/v2 v2.1.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -57,6 +59,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -75,6 +78,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/philhofer/fwd v1.1.2 // indirect + github.com/pierrec/lz4/v4 v4.1.8 // indirect github.com/pyroscope-io/dotnetdiag v1.2.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect @@ -86,6 +90,8 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/xitongsys/parquet-go v1.6.2 // indirect + github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opentelemetry.io/otel v1.15.1 // indirect go.opentelemetry.io/otel/trace v1.15.1 // indirect @@ -94,6 +100,7 @@ require ( golang.org/x/term v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect diff --git a/go.sum b/go.sum index 9e653aaa7c9c..98230e9807a7 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,11 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 h1:X8MJ0fnN5FPdcGF5Ij2/OW+HgiJrRg3AfHAx1PJtIzM= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= +github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= +github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aquasecurity/libbpfgo v0.5.0-libbpf-1.2 h1:Yywi9wC3GPDOgR8wr6P9geY2qvFqKxH5sctMOssw+MQ= github.com/aquasecurity/libbpfgo v0.5.0-libbpf-1.2/go.mod h1:0rEApF1YBHGuZ4C8OYI9q5oDBVpgqtRqYATePl9mCDk= github.com/aquasecurity/libbpfgo/helpers v0.4.6-0.20230321190037-f591a2c5734f h1:l127H3NqJBmw+XMt+haBOeZIrBppuw7TJz26cWMI9kY= @@ -81,6 +86,7 @@ github.com/aquasecurity/tracee/types v0.0.0-20231013014739-b32a168ee6a8/go.mod h github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -102,6 +108,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -186,6 +193,7 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= @@ -206,6 +214,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -224,6 +233,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -231,6 +242,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/cel-go v0.16.1 h1:3hZfSNiAU3KOiNtxuFXVp5WFy4hf/Ly3Sa4/7F8SXNo= github.com/google/cel-go v0.16.1/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= @@ -289,6 +301,7 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -307,7 +320,9 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -320,6 +335,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -397,11 +414,14 @@ github.com/opencontainers/runtime-spec v1.1.0-rc.2 h1:ucBtEms2tamYYW/SvGpvq9yUN0 github.com/opencontainers/runtime-spec v1.1.0-rc.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= +github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -447,6 +467,7 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -466,6 +487,7 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -497,6 +519,12 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= +github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM= +github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA= +github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/yashtewari/glob-intersection v0.1.0 h1:6gJvMYQlTDOL3dMsPF6J0+26vwX9MB8/1q3uAdhmTrg= github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -531,6 +559,7 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -782,6 +811,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -902,6 +933,11 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/cmd/cobra/config.go b/pkg/cmd/cobra/config.go index 07fef6620a57..3e241e3b3dca 100644 --- a/pkg/cmd/cobra/config.go +++ b/pkg/cmd/cobra/config.go @@ -352,6 +352,7 @@ type OutputConfig struct { TableVerbose OutputFormatConfig `mapstructure:"table-verbose"` JSON OutputFormatConfig `mapstructure:"json"` Gob OutputFormatConfig `mapstructure:"gob"` + Parquet OutputFormatConfig `mapstructure:"parquet"` GoTemplate OutputGoTemplateConfig `mapstructure:"gotemplate"` Forwards map[string]OutputForwardConfig `mapstructure:"forward"` Webhooks map[string]OutputWebhookConfig `mapstructure:"webhook"` @@ -392,6 +393,7 @@ func (c *OutputConfig) flags() []string { "table-verbose": c.TableVerbose.Files, "json": c.JSON.Files, "gob": c.Gob.Files, + "parquet": c.Parquet.Files, } for format, files := range formatFilesMap { for _, file := range files { diff --git a/pkg/cmd/flags/output.go b/pkg/cmd/flags/output.go index 71342fcd87c5..7d487c77ee66 100644 --- a/pkg/cmd/flags/output.go +++ b/pkg/cmd/flags/output.go @@ -47,7 +47,7 @@ func PrepareOutput(outputSlice []string, newBinary bool) (PrepareOutputResult, e return outConfig, errors.New("none output does not support path. Use '--output help' for more info") } printerMap["stdout"] = "ignore" - case "table", "table-verbose", "json", "gob": + case "table", "table-verbose", "json", "gob", "parquet": err := parseFormat(outputParts, printerMap, newBinary) if err != nil { return outConfig, err diff --git a/pkg/cmd/printer/printer.go b/pkg/cmd/printer/printer.go index 752b3d897ac4..ef853a2125ca 100644 --- a/pkg/cmd/printer/printer.go +++ b/pkg/cmd/printer/printer.go @@ -14,6 +14,8 @@ import ( "time" forward "github.com/IBM/fluent-forward-go/fluent/client" + "github.com/xitongsys/parquet-go/parquet" + pqwriter "github.com/xitongsys/parquet-go/writer" "github.com/aquasecurity/tracee/pkg/config" "github.com/aquasecurity/tracee/pkg/errfmt" @@ -68,6 +70,10 @@ func New(cfg config.PrinterConfig) (EventPrinter, error) { res = &gobEventPrinter{ out: cfg.OutFile, } + case kind == "parquet": + res = &parquetEventPrinter{ + out: cfg.OutFile, + } case kind == "forward": res = &forwardEventPrinter{ outPath: cfg.OutPath, @@ -466,6 +472,42 @@ func (p *gobEventPrinter) Epilogue(stats metrics.Stats) {} func (p gobEventPrinter) Close() { } +type parquetEventPrinter struct { + out io.WriteCloser + pqOutWriter *pqwriter.ParquetWriter +} + +func (p *parquetEventPrinter) Init() error { + // Create a new Parquet writer. + pqOutWriter, err := pqwriter.NewParquetWriterFromWriter(p.out, new(ParquetEvent), 4) + if err != nil { + logger.Errorw("Error creating Parquet writer: ", err) + return err + } + + // I chose to use gzip because its output is smaller. + pqOutWriter.CompressionType = parquet.CompressionCodec_GZIP + p.pqOutWriter = pqOutWriter + + return nil +} + +func (p *parquetEventPrinter) Preamble() {} + +func (p *parquetEventPrinter) Print(event trace.Event) { + if err := p.pqOutWriter.Write(ToParquetEvent(event)); err != nil { + logger.Errorw("Error encoding event to parquet", "error", err, "event", event) + } +} + +func (p *parquetEventPrinter) Epilogue(stats metrics.Stats) {} + +func (p parquetEventPrinter) Close() { + if err := p.pqOutWriter.WriteStop(); err != nil { + logger.Errorw("WriteStop error", "error", err) + } +} + // ignoreEventPrinter ignores events type ignoreEventPrinter struct{} diff --git a/pkg/cmd/printer/printer_parquet.go b/pkg/cmd/printer/printer_parquet.go new file mode 100644 index 000000000000..c152b18b196c --- /dev/null +++ b/pkg/cmd/printer/printer_parquet.go @@ -0,0 +1,178 @@ +package printer + +import ( + "encoding/json" + "fmt" + + "github.com/aquasecurity/tracee/types/trace" +) + +/* +ParquetEvent is used to define a schema for the parquet format. +We need to use a different structure than trace.Event because not all member types are supported by Parquet: + - unit is not supported and must be converted to an int. + - Value interface{} inside Argument is also not supported, it has no definite type + +Event Conversion to ParquetEvent is required +*/ +type ParquetEvent struct { + Timestamp int `parquet:"name=timestamp, type=INT64, logicaltype=TIMESTAMP, logicaltype.isadjustedtoutc=false, logicaltype.unit=NANOS"` + ThreadStartTime int `parquet:"name=threadStartTime, type=INT64"` + ProcessorID int `parquet:"name=processorId, type=INT64"` + ProcessID int `parquet:"name=processId, type=INT64"` + CgroupID int `parquet:"name=cgroupId, type=INT64, convertedtype=UINT_64"` + ThreadID int `parquet:"name=threadId, type=INT64"` + ParentProcessID int `parquet:"name=parentProcessId, type=INT64"` + HostProcessID int `parquet:"name=hostProcessId, type=INT64"` + HostThreadID int `parquet:"name=hostThreadId, type=INT64"` + HostParentProcessID int `parquet:"name=hostParentProcessId, type=INT64"` + UserID int `parquet:"name=UserID, type=INT64"` + MountNS int `parquet:"name=mountNamespace, type=INT64"` + PIDNS int `parquet:"name=pidNamespace, type=INT64"` + ProcessName string `parquet:"name=processName, type=BYTE_ARRAY"` + Executable ParquetFile `parquet:"name=executable"` + HostName string `parquet:"name=hostName, type=BYTE_ARRAY, convertedtype=UTF8"` + ContainerID string `parquet:"name=containerId, type=BYTE_ARRAY, convertedtype=UTF8"` + Container ParquetContainer `parquet:"name=container"` + Kubernetes ParquetKubernetes `parquet:"name=kubernetes"` + EventID int `parquet:"name=eventId, type=INT64"` + EventName string `parquet:"name=eventName, type=BYTE_ARRAY, convertedtype=UTF8"` + MatchedPolicies []string `parquet:"name=matchedPolicies, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` + ArgsNum int `parquet:"name=argsNum, type=INT64"` + ReturnValue int `parquet:"name=returnValue, type=INT64"` + Syscall string `parquet:"name=syscall, type=BYTE_ARRAY, convertedtype=UTF8"` + StackAddresses []int64 `parquet:"name=stackAddresses, type=MAP, convertedtype=LIST, valuetype=INT64, valueconvertedtype=UNIT_64"` + ContextFlags ParquetContextFlags `parquet:"name=contextFlags"` + ThreadEntityId int32 `parquet:"name=threadEntityId, type=INT32, convertedtype=UINT_32"` + ProcessEntityId int32 `parquet:"name=processEntityId, type=INT32, convertedtype=UINT_32"` + ParentEntityId int32 `parquet:"name=parentEntityId, type=INT32, convertedtype=UINT_32"` + Args string `parquet:"name=args, type=BYTE_ARRAY, convertedtype=JSON"` +} + +/* +The following structs are not necessary: ParquetContainer, ParquetKubernetes, ParquetContextFlags, ParquetFile. + +The original structs in trace.go can be changed so they will have both json and parquet tag. +For example: + + type File struct { + Path string `json:"path" parquet:"name=path, type=BYTE_ARRAY, convertedtype=UTF8"` + } + + type Container struct { + ID string `json:"id,omitempty" parquet:"name=id, type=BYTE_ARRAY, convertedtype=UTF8"` + Name string `json:"name,omitempty" parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` + ImageName string `json:"image,omitempty" parquet:"name=image, type=BYTE_ARRAY, convertedtype=UTF8"` + ImageDigest string `json:"imageDigest,omitempty" parquet:"name=imageDigest, type=BYTE_ARRAY, convertedtype=UTF8"` + } +*/ +type ParquetFile struct { + Path string `parquet:"name=path, type=BYTE_ARRAY, convertedtype=UTF8"` +} + +func toParquetFile(file trace.File) ParquetFile { + return ParquetFile{ + Path: file.Path, + } +} + +type ParquetContainer struct { + ID string `parquet:"name=id, type=BYTE_ARRAY, convertedtype=UTF8"` + Name string `parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` + ImageName string `parquet:"name=image, type=BYTE_ARRAY, convertedtype=UTF8"` + ImageDigest string `parquet:"name=imageDigest, type=BYTE_ARRAY, convertedtype=UTF8"` +} + +func toParquetContainer(container trace.Container) ParquetContainer { + return ParquetContainer{ + ID: container.ID, + Name: container.Name, + ImageName: container.ImageName, + ImageDigest: container.ImageDigest, + } +} + +type ParquetKubernetes struct { + PodName string `parquet:"name=podName, type=BYTE_ARRAY, convertedtype=UTF8"` + PodNamespace string `parquet:"name=podNamespace, type=BYTE_ARRAY, convertedtype=UTF8"` + PodUID string `parquet:"name=podUID, type=BYTE_ARRAY, convertedtype=UTF8"` + PodSandbox bool `parquet:"name=podSandbox, type=BOOLEAN"` +} + +func toParquetKubernetes(kubernetes trace.Kubernetes) ParquetKubernetes { + return ParquetKubernetes{ + PodName: kubernetes.PodName, + PodNamespace: kubernetes.PodNamespace, + PodUID: kubernetes.PodUID, + PodSandbox: kubernetes.PodSandbox, + } +} + +type ParquetContextFlags struct { + ContainerStarted bool `parquet:"name=containerStarted, type=BOOLEAN"` + IsCompat bool `parquet:"name=isCompat, type=BOOLEAN"` +} + +func toParquetContextFlags(ctxFlags trace.ContextFlags) ParquetContextFlags { + return ParquetContextFlags{ + ContainerStarted: ctxFlags.ContainerStarted, + IsCompat: ctxFlags.IsCompat, + } +} + +func toInt64Slice(uint64Slice []uint64) []int64 { + int64Slice := make([]int64, len(uint64Slice)) + for i, val := range uint64Slice { + int64Slice[i] = int64(val) + } + return int64Slice +} + +func toJsonStr(toJson interface{}) string { + + argsJsonByte, err := json.Marshal(toJson) + if err != nil { + fmt.Println("Error marshaling to JSON:", err) + return "" + } + return string(argsJsonByte) +} + +func ToParquetEvent(event trace.Event) ParquetEvent { + + pqEvent := ParquetEvent{ + Timestamp: event.Timestamp, + ThreadStartTime: event.ThreadStartTime, + ProcessorID: event.ProcessorID, + ProcessID: event.ProcessID, + CgroupID: int(event.CgroupID), + ThreadID: event.ThreadID, + ParentProcessID: event.ParentProcessID, + HostProcessID: event.HostProcessID, + HostThreadID: event.HostThreadID, + HostParentProcessID: event.HostParentProcessID, + UserID: event.UserID, + MountNS: event.MountNS, + PIDNS: event.PIDNS, + ProcessName: event.ProcessName, + Executable: toParquetFile(event.Executable), + HostName: event.HostName, + ContainerID: event.ContainerID, + Container: toParquetContainer(event.Container), + Kubernetes: toParquetKubernetes(event.Kubernetes), + EventID: event.EventID, + EventName: event.EventName, + MatchedPolicies: event.MatchedPolicies, + ArgsNum: event.ArgsNum, + ReturnValue: event.ReturnValue, + Syscall: event.Syscall, + StackAddresses: toInt64Slice(event.StackAddresses), + ContextFlags: toParquetContextFlags(event.ContextFlags), + ThreadEntityId: int32(event.ThreadEntityId), + ProcessEntityId: int32(event.ProcessEntityId), + ParentEntityId: int32(event.ParentEntityId), + Args: toJsonStr(event.Args), + } + + return pqEvent +} From 5c0c06e9cf0b0c61d8276f3d2df538b58c5b60f0 Mon Sep 17 00:00:00 2001 From: Shani Date: Mon, 13 Nov 2023 20:29:30 +0200 Subject: [PATCH 2/2] Fix lint errors --- pkg/cmd/printer/printer_parquet.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/cmd/printer/printer_parquet.go b/pkg/cmd/printer/printer_parquet.go index c152b18b196c..60cd6b8f0fa5 100644 --- a/pkg/cmd/printer/printer_parquet.go +++ b/pkg/cmd/printer/printer_parquet.go @@ -129,7 +129,6 @@ func toInt64Slice(uint64Slice []uint64) []int64 { } func toJsonStr(toJson interface{}) string { - argsJsonByte, err := json.Marshal(toJson) if err != nil { fmt.Println("Error marshaling to JSON:", err) @@ -139,7 +138,6 @@ func toJsonStr(toJson interface{}) string { } func ToParquetEvent(event trace.Event) ParquetEvent { - pqEvent := ParquetEvent{ Timestamp: event.Timestamp, ThreadStartTime: event.ThreadStartTime,