diff --git a/.gitignore b/.gitignore index 5c84136..b50398d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ build dist *.egg-info *.antlr +.vscode +*.ipynb_checkpoints/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f73893..f1c9842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,26 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). > - **Fixed**: for any bug fixes. > - **Security**: in case of vulnerabilities. -## [[UNRELEASED](https://github.com/sysflow-telemetry/sf-apis/compare/0.1-rc3...HEAD)] +## [[UNRELEASED](https://github.com/sysflow-telemetry/sf-apis/compare/0.1.0-rc4...HEAD)] + +## [[0.1.0-rc4](https://github.com/sysflow-telemetry/sf-apis/compare/0.1-rc3...0.1.0-rc4)] - 2020-08-10 + +### Added + +- Added `node.id`, `node.ip`, `proc.entry`, and `schema` attributes to query language and export APIs. +- Added golang APIs. + +### Changed + +- Support for new Avro schema (version 2). +- Added missing EXIT opflag to Python APIs. +- Adding patch level to comply with semnatic versioning. + +### Fixed + +- Fixed open flags bitmaps. +- Fixed attribute name typo when computing proc and pproc duration. +- Fixed bug in provenance queries. ## [[0.1-rc3](https://github.com/sysflow-telemetry/sf-apis/compare/0.1-rc2...0.1-rc3)] - 2020-03-17 diff --git a/Dockerfile.sfnb b/Dockerfile.sfnb index ebef2ff..ce4dd8c 100644 --- a/Dockerfile.sfnb +++ b/Dockerfile.sfnb @@ -1,7 +1,6 @@ -FROM jupyter/minimal-notebook +FROM jupyter/scipy-notebook -# Install pandas, numpy, and graphviz python bindings -RUN pip install graphviz pandas numpy matplotlib +# Install graphviz python bindings RUN conda install -y graphviz # Change user to root @@ -14,12 +13,13 @@ RUN apt-get update -yqq && \ apt-get clean -yqq && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/lib/apt/archive/* -# Install sysflow API -COPY py3 /build -RUN cd /build && python setup.py install && rm -r /build +# Copy sources +COPY --chown=$NB_UID:$NB_UID py3 /tmp/build # Switch back to jovyan to avoid accidental container runs as root USER $NB_UID -# Copy example scripts into container -#COPY pynb /home/$NB_USER/work +# Install sysflow API +RUN cd /tmp/build && pip install . && rm -r /tmp/build + + diff --git a/Dockerfile.sysprint b/Dockerfile.sysprint index d6753b3..c87802c 100644 --- a/Dockerfile.sysprint +++ b/Dockerfile.sysprint @@ -20,8 +20,10 @@ FROM registry.access.redhat.com/ubi8/ubi # Install Python environment RUN dnf install -y --disableplugin=subscription-manager \ + gcc \ python3 \ - python3-wheel && \ + python3-devel \ + python3-wheel && \ dnf -y clean all && rm -rf /var/cache/dnf && \ mkdir -p /usr/local/lib/python3.6/site-packages && \ ln -s /usr/bin/easy_install-3 /usr/bin/easy_install diff --git a/README.md b/README.md index 2f3e1e5..2496c4a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ # Supported tags and respective `Dockerfile` links -- [`0.1-rc3`](https://github.com/sysflow-telemetry/sf-exporter/blob/0.1-rc3/Dockerfile), [`latest`](https://github.com/sysflow-telemetry/sf-apis/blob/master/Dockerfile) +- [`0.1.0-rc4`](https://github.com/sysflow-telemetry/sf-exporter/blob/0.1.0-rc4/Dockerfile), [`latest`](https://github.com/sysflow-telemetry/sf-apis/blob/master/Dockerfile) # Quick reference diff --git a/avro/avdl/entity/file.avdl b/avro/avdl/entity/file.avdl index 290c584..d95f180 100644 --- a/avro/avdl/entity/file.avdl +++ b/avro/avdl/entity/file.avdl @@ -23,7 +23,7 @@ import idl "container.avdl"; record File { sysflow.type.SFObjectState state; //state of file - created, modified, reupped sysflow.type.FOID oid; // hash file id, container id, into 128 bit value. - timestamp_ms ts; + long ts; int restype; string path; union{null, string} containerId; diff --git a/avro/avdl/entity/header.avdl b/avro/avdl/entity/header.avdl index ffff57c..ae9764c 100644 --- a/avro/avdl/entity/header.avdl +++ b/avro/avdl/entity/header.avdl @@ -19,7 +19,8 @@ @namespace("sysflow.entity") protocol SysFlow { record SFHeader { - long version; - string exporter; + long version = 2; + string exporter; + string ip = "NA"; } } diff --git a/avro/avdl/entity/process.avdl b/avro/avdl/entity/process.avdl index d044280..5c926cf 100644 --- a/avro/avdl/entity/process.avdl +++ b/avro/avdl/entity/process.avdl @@ -24,7 +24,7 @@ record Process { sysflow.type.SFObjectState state; //sysflow process object state.. created, modified, reupped sysflow.type.OID oid; // monotonic process creation time + host PID union{null, sysflow.type.OID} poid; - timestamp_ms ts; + long ts; //int hpid; string exe; string exeArgs; @@ -38,5 +38,6 @@ record Process { //int threadCount; //int childCount; union{null, string} containerId; + boolean entry = false; } } diff --git a/avro/avdl/event/fileevent.avdl b/avro/avdl/event/fileevent.avdl index 8ad1ce5..658f022 100644 --- a/avro/avdl/event/fileevent.avdl +++ b/avro/avdl/event/fileevent.avdl @@ -21,7 +21,7 @@ protocol SysFlow { import idl "../type/datatypes.avdl"; record FileEvent { sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID - timestamp_ms ts; + long ts; long tid; int opFlags; sysflow.type.FOID fileOID; diff --git a/avro/avdl/event/networkevent.avdl b/avro/avdl/event/networkevent.avdl index dbc2c01..baf1352 100644 --- a/avro/avdl/event/networkevent.avdl +++ b/avro/avdl/event/networkevent.avdl @@ -21,7 +21,7 @@ protocol SysFlow { import idl "../type/datatypes.avdl"; record NetworkEvent { sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID - timestamp_ms ts; + long ts; long tid; int opFlags; int sip; diff --git a/avro/avdl/event/processevent.avdl b/avro/avdl/event/processevent.avdl index 590b37b..d020c29 100644 --- a/avro/avdl/event/processevent.avdl +++ b/avro/avdl/event/processevent.avdl @@ -21,7 +21,7 @@ protocol SysFlow { import idl "../type/datatypes.avdl"; record ProcessEvent { sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID - timestamp_ms ts; + long ts; long tid; int opFlags; array args; diff --git a/avro/avdl/flow/fileflow.avdl b/avro/avdl/flow/fileflow.avdl index 5db6b24..9aa5d78 100644 --- a/avro/avdl/flow/fileflow.avdl +++ b/avro/avdl/flow/fileflow.avdl @@ -21,11 +21,11 @@ protocol SysFlow { import idl "../type/datatypes.avdl"; record FileFlow { sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID - timestamp_ms ts; + long ts; long tid; int opFlags; int openFlags; - timestamp_ms endTs; + long endTs; sysflow.type.FOID fileOID; int fd; long numRRecvOps; diff --git a/avro/avdl/flow/networkflow.avdl b/avro/avdl/flow/networkflow.avdl index 9fa0e27..b6968b2 100644 --- a/avro/avdl/flow/networkflow.avdl +++ b/avro/avdl/flow/networkflow.avdl @@ -21,10 +21,10 @@ protocol SysFlow { import idl "../type/datatypes.avdl"; record NetworkFlow { sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID - timestamp_ms ts; + long ts; long tid; int opFlags; - timestamp_ms endTs; + long endTs; int sip; int sport; int dip; diff --git a/avro/avdl/flow/processflow.avdl b/avro/avdl/flow/processflow.avdl index fa370b8..2334428 100644 --- a/avro/avdl/flow/processflow.avdl +++ b/avro/avdl/flow/processflow.avdl @@ -21,11 +21,11 @@ protocol SysFlow { import idl "../type/datatypes.avdl"; record ProcessFlow { sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID - timestamp_ms ts; - long tid; + long ts; + long numThreadsCloned; int opFlags; - timestamp_ms endTs; - array args; - int ret; + long endTs; + long numThreadsExited; + long numCloneErrors; } } diff --git a/avro/avdl/sysflow.avdl b/avro/avdl/sysflow.avdl index 1043f1a..b364008 100644 --- a/avro/avdl/sysflow.avdl +++ b/avro/avdl/sysflow.avdl @@ -30,6 +30,6 @@ import idl "entity/container.avdl"; import idl "entity/file.avdl"; record SysFlow { - union {sysflow.entity.SFHeader, sysflow.entity.Container, sysflow.entity.Process, sysflow.entity.File, sysflow.event.ProcessEvent, sysflow.flow.NetworkFlow, sysflow.flow.FileFlow, sysflow.event.FileEvent, sysflow.event.NetworkEvent} rec; + union {sysflow.entity.SFHeader, sysflow.entity.Container, sysflow.entity.Process, sysflow.entity.File, sysflow.event.ProcessEvent, sysflow.flow.NetworkFlow, sysflow.flow.FileFlow, sysflow.event.FileEvent, sysflow.event.NetworkEvent, sysflow.flow.ProcessFlow} rec; } } diff --git a/avro/avdl/type/datatypes.avdl b/avro/avdl/type/datatypes.avdl index 2c68ea6..93fa984 100644 --- a/avro/avdl/type/datatypes.avdl +++ b/avro/avdl/type/datatypes.avdl @@ -19,41 +19,22 @@ @namespace ("sysflow.type") protocol SysFlow { -//fixed OID(16); record OID { - timestamp_ms createTS; + long createTS; long hpid; } -/*record FOID { - long bits0; - long bits8; - long bits16; -}*/ - fixed FOID(20); fixed ContainerID(6); -/*enum EventType { - CLONE, - EXEC, - EXIT - }*/ enum SFObjectState { CREATED, MODIFIED, REUP - } -/* -enum ResourceType { - SF_FILE, - SF_DIRECTORY, - SF_PIPE, - SF_UNIX - }*/ +} enum ContainerType { CT_DOCKER, @@ -63,9 +44,9 @@ enum ContainerType { CT_RKT, CT_CUSTOM, CT_CRI, - CT_CONTAINERD, - CT_CRIO, - CT_BPM + CT_CONTAINERD, + CT_CRIO, + CT_BPM } diff --git a/avro/avpr/sysflow.avpr b/avro/avpr/sysflow.avpr index 3c2e00d..ec6f897 100644 --- a/avro/avpr/sysflow.avpr +++ b/avro/avpr/sysflow.avpr @@ -8,10 +8,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -45,31 +42,22 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { - "name" : "tid", + "name" : "numThreadsCloned", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { - "name" : "args", - "type" : { - "type" : "array", - "items" : "string" - } + "name" : "numThreadsExited", + "type" : "long" }, { - "name" : "ret", - "type" : "int" + "name" : "numCloneErrors", + "type" : "long" } ] }, { "type" : "record", @@ -80,10 +68,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -92,10 +77,7 @@ "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "sip", "type" : "int" @@ -136,10 +118,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -151,10 +130,7 @@ "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "fileOID", "type" : "sysflow.type.FOID" @@ -183,10 +159,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -212,10 +185,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -250,10 +220,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -308,10 +275,7 @@ "type" : [ "null", "sysflow.type.OID" ] }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "exe", "type" : "string" @@ -336,6 +300,10 @@ }, { "name" : "containerId", "type" : [ "null", "string" ] + }, { + "name" : "entry", + "type" : "boolean", + "default" : false } ] }, { "type" : "record", @@ -343,10 +311,15 @@ "namespace" : "sysflow.entity", "fields" : [ { "name" : "version", - "type" : "long" + "type" : "long", + "default" : 2 }, { "name" : "exporter", "type" : "string" + }, { + "name" : "ip", + "type" : "string", + "default" : "NA" } ] }, { "type" : "record", @@ -360,10 +333,7 @@ "type" : "sysflow.type.FOID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "restype", "type" : "int" @@ -379,7 +349,7 @@ "name" : "SysFlow", "fields" : [ { "name" : "rec", - "type" : [ "sysflow.entity.SFHeader", "sysflow.entity.Container", "sysflow.entity.Process", "sysflow.entity.File", "sysflow.event.ProcessEvent", "sysflow.flow.NetworkFlow", "sysflow.flow.FileFlow", "sysflow.event.FileEvent", "sysflow.event.NetworkEvent" ] + "type" : [ "sysflow.entity.SFHeader", "sysflow.entity.Container", "sysflow.entity.Process", "sysflow.entity.File", "sysflow.event.ProcessEvent", "sysflow.flow.NetworkFlow", "sysflow.flow.FileFlow", "sysflow.event.FileEvent", "sysflow.event.NetworkEvent", "sysflow.flow.ProcessFlow" ] } ] } ], "messages" : { } diff --git a/avro/avsc/File.avsc b/avro/avsc/File.avsc index aef04cb..aa08743 100644 --- a/avro/avsc/File.avsc +++ b/avro/avsc/File.avsc @@ -20,10 +20,7 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "restype", "type" : "int" diff --git a/avro/avsc/FileEvent.avsc b/avro/avsc/FileEvent.avsc index 8ea0b6b..6acc782 100644 --- a/avro/avsc/FileEvent.avsc +++ b/avro/avsc/FileEvent.avsc @@ -10,10 +10,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -21,10 +18,7 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" diff --git a/avro/avsc/FileFlow.avsc b/avro/avsc/FileFlow.avsc index 2f81619..e67e10d 100644 --- a/avro/avsc/FileFlow.avsc +++ b/avro/avsc/FileFlow.avsc @@ -10,10 +10,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -21,10 +18,7 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -36,10 +30,7 @@ "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "fileOID", "type" : { diff --git a/avro/avsc/NetworkEvent.avsc b/avro/avsc/NetworkEvent.avsc index 8561610..210dd79 100644 --- a/avro/avsc/NetworkEvent.avsc +++ b/avro/avsc/NetworkEvent.avsc @@ -10,10 +10,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -21,10 +18,7 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" diff --git a/avro/avsc/NetworkFlow.avsc b/avro/avsc/NetworkFlow.avsc index 36818a1..21c7a9a 100644 --- a/avro/avsc/NetworkFlow.avsc +++ b/avro/avsc/NetworkFlow.avsc @@ -10,10 +10,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -21,10 +18,7 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -33,10 +27,7 @@ "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "sip", "type" : "int" diff --git a/avro/avsc/OID.avsc b/avro/avsc/OID.avsc index d710b4e..3df5084 100644 --- a/avro/avsc/OID.avsc +++ b/avro/avsc/OID.avsc @@ -4,10 +4,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" diff --git a/avro/avsc/Process.avsc b/avro/avsc/Process.avsc index 7ccac20..cfc5daa 100644 --- a/avro/avsc/Process.avsc +++ b/avro/avsc/Process.avsc @@ -18,10 +18,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -32,10 +29,7 @@ "type" : [ "null", "sysflow.type.OID" ] }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "exe", "type" : "string" @@ -60,5 +54,9 @@ }, { "name" : "containerId", "type" : [ "null", "string" ] + }, { + "name" : "entry", + "type" : "boolean", + "default" : false } ] } diff --git a/avro/avsc/ProcessEvent.avsc b/avro/avsc/ProcessEvent.avsc index b2538f1..ff0b204 100644 --- a/avro/avsc/ProcessEvent.avsc +++ b/avro/avsc/ProcessEvent.avsc @@ -10,10 +10,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -21,10 +18,7 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" diff --git a/avro/avsc/ProcessFlow.avsc b/avro/avsc/ProcessFlow.avsc index 2805d84..9e4691c 100644 --- a/avro/avsc/ProcessFlow.avsc +++ b/avro/avsc/ProcessFlow.avsc @@ -10,10 +10,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -21,30 +18,21 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { - "name" : "tid", + "name" : "numThreadsCloned", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { - "name" : "args", - "type" : { - "type" : "array", - "items" : "string" - } + "name" : "numThreadsExited", + "type" : "long" }, { - "name" : "ret", - "type" : "int" + "name" : "numCloneErrors", + "type" : "long" } ] } diff --git a/avro/avsc/SFHeader.avsc b/avro/avsc/SFHeader.avsc index d9aca16..1d237cd 100644 --- a/avro/avsc/SFHeader.avsc +++ b/avro/avsc/SFHeader.avsc @@ -4,9 +4,14 @@ "namespace" : "sysflow.entity", "fields" : [ { "name" : "version", - "type" : "long" + "type" : "long", + "default" : 2 }, { "name" : "exporter", "type" : "string" + }, { + "name" : "ip", + "type" : "string", + "default" : "NA" } ] } diff --git a/avro/avsc/SysFlow.avsc b/avro/avsc/SysFlow.avsc index 589ba07..c05a656 100644 --- a/avro/avsc/SysFlow.avsc +++ b/avro/avsc/SysFlow.avsc @@ -10,10 +10,15 @@ "namespace" : "sysflow.entity", "fields" : [ { "name" : "version", - "type" : "long" + "type" : "long", + "default" : 2 }, { "name" : "exporter", "type" : "string" + }, { + "name" : "ip", + "type" : "string", + "default" : "NA" } ] }, { "type" : "record", @@ -63,10 +68,7 @@ "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "hpid", "type" : "long" @@ -77,10 +79,7 @@ "type" : [ "null", "sysflow.type.OID" ] }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "exe", "type" : "string" @@ -105,6 +104,10 @@ }, { "name" : "containerId", "type" : [ "null", "string" ] + }, { + "name" : "entry", + "type" : "boolean", + "default" : false } ] }, { "type" : "record", @@ -123,10 +126,7 @@ } }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "restype", "type" : "int" @@ -146,10 +146,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -175,10 +172,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -187,10 +181,7 @@ "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "sip", "type" : "int" @@ -231,10 +222,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -246,10 +234,7 @@ "type" : "int" }, { "name" : "endTs", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "fileOID", "type" : "sysflow.type.FOID" @@ -278,10 +263,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -307,10 +289,7 @@ "type" : "sysflow.type.OID" }, { "name" : "ts", - "type" : { - "type" : "long", - "logicalType" : "timestamp-millis" - } + "type" : "long" }, { "name" : "tid", "type" : "long" @@ -336,6 +315,32 @@ "name" : "ret", "type" : "int" } ] + }, { + "type" : "record", + "name" : "ProcessFlow", + "namespace" : "sysflow.flow", + "fields" : [ { + "name" : "procOID", + "type" : "sysflow.type.OID" + }, { + "name" : "ts", + "type" : "long" + }, { + "name" : "numThreadsCloned", + "type" : "long" + }, { + "name" : "opFlags", + "type" : "int" + }, { + "name" : "endTs", + "type" : "long" + }, { + "name" : "numThreadsExited", + "type" : "long" + }, { + "name" : "numCloneErrors", + "type" : "long" + } ] } ] } ] } diff --git a/avro/generateCClasses.sh b/avro/generateCClasses.sh old mode 100644 new mode 100755 index 134caad..7425912 --- a/avro/generateCClasses.sh +++ b/avro/generateCClasses.sh @@ -1,6 +1,6 @@ #!/bin/bash -java -jar avro-tools/avro-tools-1.9.1.jar idl avdl/sysflow.avdl ./avpr/sysflow.avpr -java -jar avro-tools/avro-tools-1.9.1.jar idl2schemata ./avdl/sysflow.avdl avsc/ +java -jar avro-tools/avro-tools-1.10.0.jar idl avdl/sysflow.avdl ./avpr/sysflow.avpr +java -jar avro-tools/avro-tools-1.10.0.jar idl2schemata ./avdl/sysflow.avdl avsc/ # cpp stub generation #avrogencpp -i ./avsc/ContainerID.avsc -o ./src/sysflow/container_id.hh -n sysflow.type @@ -13,4 +13,14 @@ java -jar avro-tools/avro-tools-1.9.1.jar idl2schemata ./avdl/sysflow.avdl avsc/ #avrogencpp -i ./avsc/FileFlow.avsc -o ./src/sysflow/file_flow.hh -n sysflow.flow #avrogencpp -i ./avsc/NetworkFlow.avsc -o ./src/sysflow/network_flow.hh -n sysflow.flow #avrogencpp -i ./avsc/ActionType.avsc -o ./src/sysflow/action_type.hh -n sysflow.type -#avrogencpp -i ./avsc/SysFlow.avsc -o ./src/sysflow/sysflow.hh -n sysflow +avrogencpp -i ./avsc/SysFlow.avsc -o ../c++/sysflow/sysflow.hh -n sysflow +echo "#ifndef __AVSC_SYSFLOW${1}" > ../c++/sysflow/avsc_sysflow${1}.hh +echo "#define __AVSC_SYSFLOW${1}" >> ../c++/sysflow/avsc_sysflow${1}.hh +echo "#include " >> ../c++/sysflow/avsc_sysflow${1}.hh +echo -n "extern const std::string AVSC_SF = " >> ../c++/sysflow/avsc_sysflow${1}.hh +#AVSC=`cat avsc/sysflow${1}/TCCDMDatum.avsc` +#sed -e "s/\"/\\\\\"/gi" avsc/sysflow${1}/TCCDMDatum.avsc | tr -d '\n' >> ../c++/sysflow/avsc_sysflow${1}.hh +cat ./avsc/SysFlow.avsc | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' | tr -d '\n' >> ../c++/sysflow/avsc_sysflow${1}.hh +#printf "%q" $AVSC >> ../c++/sysflow/avsc_sysflow${1}.hh +echo ";" >> ../c++/sysflow/avsc_sysflow${1}.hh +echo "#endif" >> ../c++/sysflow/avsc_sysflow${1}.hh diff --git a/c++/sysflow/avsc_sysflow2.hh b/c++/sysflow/avsc_sysflow2.hh new file mode 100644 index 0000000..4994596 --- /dev/null +++ b/c++/sysflow/avsc_sysflow2.hh @@ -0,0 +1,5 @@ +#ifndef __AVSC_SYSFLOW2 +#define __AVSC_SYSFLOW2 +#include +extern const std::string AVSC_SF = "{\n \"type\" : \"record\",\n \"name\" : \"SysFlow\",\n \"namespace\" : \"sysflow\",\n \"fields\" : [ {\n \"name\" : \"rec\",\n \"type\" : [ {\n \"type\" : \"record\",\n \"name\" : \"SFHeader\",\n \"namespace\" : \"sysflow.entity\",\n \"fields\" : [ {\n \"name\" : \"version\",\n \"type\" : \"long\",\n \"default\" : 2\n }, {\n \"name\" : \"exporter\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"ip\",\n \"type\" : \"string\",\n \"default\" : \"NA\"\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"Container\",\n \"namespace\" : \"sysflow.entity\",\n \"fields\" : [ {\n \"name\" : \"id\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"name\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"image\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"imageid\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"type\",\n \"type\" : {\n \"type\" : \"enum\",\n \"name\" : \"ContainerType\",\n \"namespace\" : \"sysflow.type\",\n \"symbols\" : [ \"CT_DOCKER\", \"CT_LXC\", \"CT_LIBVIRT_LXC\", \"CT_MESOS\", \"CT_RKT\", \"CT_CUSTOM\", \"CT_CRI\", \"CT_CONTAINERD\", \"CT_CRIO\", \"CT_BPM\" ]\n }\n }, {\n \"name\" : \"privileged\",\n \"type\" : \"boolean\"\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"Process\",\n \"namespace\" : \"sysflow.entity\",\n \"fields\" : [ {\n \"name\" : \"state\",\n \"type\" : {\n \"type\" : \"enum\",\n \"name\" : \"SFObjectState\",\n \"namespace\" : \"sysflow.type\",\n \"symbols\" : [ \"CREATED\", \"MODIFIED\", \"REUP\" ]\n }\n }, {\n \"name\" : \"oid\",\n \"type\" : {\n \"type\" : \"record\",\n \"name\" : \"OID\",\n \"namespace\" : \"sysflow.type\",\n \"fields\" : [ {\n \"name\" : \"createTS\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"hpid\",\n \"type\" : \"long\"\n } ]\n }\n }, {\n \"name\" : \"poid\",\n \"type\" : [ \"null\", \"sysflow.type.OID\" ]\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"exe\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"exeArgs\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"uid\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"userName\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"gid\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"groupName\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"tty\",\n \"type\" : \"boolean\"\n }, {\n \"name\" : \"containerId\",\n \"type\" : [ \"null\", \"string\" ]\n }, {\n \"name\" : \"entry\",\n \"type\" : \"boolean\",\n \"default\" : false\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"File\",\n \"namespace\" : \"sysflow.entity\",\n \"fields\" : [ {\n \"name\" : \"state\",\n \"type\" : \"sysflow.type.SFObjectState\"\n }, {\n \"name\" : \"oid\",\n \"type\" : {\n \"type\" : \"fixed\",\n \"name\" : \"FOID\",\n \"namespace\" : \"sysflow.type\",\n \"size\" : 20\n }\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"restype\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"path\",\n \"type\" : \"string\"\n }, {\n \"name\" : \"containerId\",\n \"type\" : [ \"null\", \"string\" ]\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"ProcessEvent\",\n \"namespace\" : \"sysflow.event\",\n \"fields\" : [ {\n \"name\" : \"procOID\",\n \"type\" : \"sysflow.type.OID\"\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"tid\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"opFlags\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"args\",\n \"type\" : {\n \"type\" : \"array\",\n \"items\" : \"string\"\n }\n }, {\n \"name\" : \"ret\",\n \"type\" : \"int\"\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"NetworkFlow\",\n \"namespace\" : \"sysflow.flow\",\n \"fields\" : [ {\n \"name\" : \"procOID\",\n \"type\" : \"sysflow.type.OID\"\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"tid\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"opFlags\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"endTs\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"sip\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"sport\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"dip\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"dport\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"proto\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"fd\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"numRRecvOps\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numWSendOps\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numRRecvBytes\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numWSendBytes\",\n \"type\" : \"long\"\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"FileFlow\",\n \"namespace\" : \"sysflow.flow\",\n \"fields\" : [ {\n \"name\" : \"procOID\",\n \"type\" : \"sysflow.type.OID\"\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"tid\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"opFlags\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"openFlags\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"endTs\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"fileOID\",\n \"type\" : \"sysflow.type.FOID\"\n }, {\n \"name\" : \"fd\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"numRRecvOps\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numWSendOps\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numRRecvBytes\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numWSendBytes\",\n \"type\" : \"long\"\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"FileEvent\",\n \"namespace\" : \"sysflow.event\",\n \"fields\" : [ {\n \"name\" : \"procOID\",\n \"type\" : \"sysflow.type.OID\"\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"tid\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"opFlags\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"fileOID\",\n \"type\" : \"sysflow.type.FOID\"\n }, {\n \"name\" : \"ret\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"newFileOID\",\n \"type\" : [ \"null\", \"sysflow.type.FOID\" ]\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"NetworkEvent\",\n \"namespace\" : \"sysflow.event\",\n \"fields\" : [ {\n \"name\" : \"procOID\",\n \"type\" : \"sysflow.type.OID\"\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"tid\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"opFlags\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"sip\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"sport\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"dip\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"dport\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"proto\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"ret\",\n \"type\" : \"int\"\n } ]\n }, {\n \"type\" : \"record\",\n \"name\" : \"ProcessFlow\",\n \"namespace\" : \"sysflow.flow\",\n \"fields\" : [ {\n \"name\" : \"procOID\",\n \"type\" : \"sysflow.type.OID\"\n }, {\n \"name\" : \"ts\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numThreadsCloned\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"opFlags\",\n \"type\" : \"int\"\n }, {\n \"name\" : \"endTs\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numThreadsExited\",\n \"type\" : \"long\"\n }, {\n \"name\" : \"numCloneErrors\",\n \"type\" : \"long\"\n } ]\n } ]\n } ]\n}\n"; +#endif diff --git a/c++/sysflow/sysflow.hh b/c++/sysflow/sysflow.hh index 1157ca3..1c1bfa3 100644 --- a/c++/sysflow/sysflow.hh +++ b/c++/sysflow/sysflow.hh @@ -31,9 +31,11 @@ namespace sysflow { struct SFHeader { int64_t version; std::string exporter; + std::string ip; SFHeader() : version(int64_t()), - exporter(std::string()) + exporter(std::string()), + ip(std::string()) { } }; @@ -133,6 +135,7 @@ struct Process { std::string groupName; bool tty; containerId_t containerId; + bool entry; Process() : state(SFObjectState()), oid(OID()), @@ -145,7 +148,8 @@ struct Process { gid(int32_t()), groupName(std::string()), tty(bool()), - containerId(containerId_t()) + containerId(containerId_t()), + entry(bool()) { } }; @@ -329,6 +333,25 @@ struct NetworkEvent { { } }; +struct ProcessFlow { + OID procOID; + int64_t ts; + int64_t numThreadsCloned; + int32_t opFlags; + int64_t endTs; + int64_t numThreadsExited; + int64_t numCloneErrors; + ProcessFlow() : + procOID(OID()), + ts(int64_t()), + numThreadsCloned(int64_t()), + opFlags(int32_t()), + endTs(int64_t()), + numThreadsExited(int64_t()), + numCloneErrors(int64_t()) + { } +}; + struct _SysFlow_avsc_Union__4__ { private: size_t idx_; @@ -353,6 +376,8 @@ public: void set_FileEvent(const FileEvent& v); NetworkEvent get_NetworkEvent() const; void set_NetworkEvent(const NetworkEvent& v); + ProcessFlow get_ProcessFlow() const; + void set_ProcessFlow(const ProcessFlow& v); _SysFlow_avsc_Union__4__(); }; @@ -546,6 +571,20 @@ void _SysFlow_avsc_Union__4__::set_NetworkEvent(const NetworkEvent& v) { value_ = v; } +inline +ProcessFlow _SysFlow_avsc_Union__4__::get_ProcessFlow() const { + if (idx_ != 9) { + throw avro::Exception("Invalid type for union"); + } + return boost::any_cast(value_); +} + +inline +void _SysFlow_avsc_Union__4__::set_ProcessFlow(const ProcessFlow& v) { + idx_ = 9; + value_ = v; +} + inline _SysFlow_avsc_Union__0__::_SysFlow_avsc_Union__0__() : idx_(0) { } inline _SysFlow_avsc_Union__1__::_SysFlow_avsc_Union__1__() : idx_(0) { } inline _SysFlow_avsc_Union__2__::_SysFlow_avsc_Union__2__() : idx_(0) { } @@ -557,6 +596,7 @@ template<> struct codec_traits { static void encode(Encoder& e, const sysflow::SFHeader& v) { avro::encode(e, v.version); avro::encode(e, v.exporter); + avro::encode(e, v.ip); } static void decode(Decoder& d, sysflow::SFHeader& v) { if (avro::ResolvingDecoder *rd = @@ -571,6 +611,9 @@ template<> struct codec_traits { case 1: avro::decode(d, v.exporter); break; + case 2: + avro::decode(d, v.ip); + break; default: break; } @@ -578,6 +621,7 @@ template<> struct codec_traits { } else { avro::decode(d, v.version); avro::decode(d, v.exporter); + avro::decode(d, v.ip); } } }; @@ -780,6 +824,7 @@ template<> struct codec_traits { avro::encode(e, v.groupName); avro::encode(e, v.tty); avro::encode(e, v.containerId); + avro::encode(e, v.entry); } static void decode(Decoder& d, sysflow::Process& v) { if (avro::ResolvingDecoder *rd = @@ -824,6 +869,9 @@ template<> struct codec_traits { case 11: avro::decode(d, v.containerId); break; + case 12: + avro::decode(d, v.entry); + break; default: break; } @@ -841,6 +889,7 @@ template<> struct codec_traits { avro::decode(d, v.groupName); avro::decode(d, v.tty); avro::decode(d, v.containerId); + avro::decode(d, v.entry); } } }; @@ -1301,6 +1350,60 @@ template<> struct codec_traits { } }; +template<> struct codec_traits { + static void encode(Encoder& e, const sysflow::ProcessFlow& v) { + avro::encode(e, v.procOID); + avro::encode(e, v.ts); + avro::encode(e, v.numThreadsCloned); + avro::encode(e, v.opFlags); + avro::encode(e, v.endTs); + avro::encode(e, v.numThreadsExited); + avro::encode(e, v.numCloneErrors); + } + static void decode(Decoder& d, sysflow::ProcessFlow& v) { + if (avro::ResolvingDecoder *rd = + dynamic_cast(&d)) { + const std::vector fo = rd->fieldOrder(); + for (std::vector::const_iterator it = fo.begin(); + it != fo.end(); ++it) { + switch (*it) { + case 0: + avro::decode(d, v.procOID); + break; + case 1: + avro::decode(d, v.ts); + break; + case 2: + avro::decode(d, v.numThreadsCloned); + break; + case 3: + avro::decode(d, v.opFlags); + break; + case 4: + avro::decode(d, v.endTs); + break; + case 5: + avro::decode(d, v.numThreadsExited); + break; + case 6: + avro::decode(d, v.numCloneErrors); + break; + default: + break; + } + } + } else { + avro::decode(d, v.procOID); + avro::decode(d, v.ts); + avro::decode(d, v.numThreadsCloned); + avro::decode(d, v.opFlags); + avro::decode(d, v.endTs); + avro::decode(d, v.numThreadsExited); + avro::decode(d, v.numCloneErrors); + } + } +}; + template<> struct codec_traits { static void encode(Encoder& e, sysflow::_SysFlow_avsc_Union__4__ v) { e.encodeUnionIndex(v.idx()); @@ -1332,11 +1435,14 @@ template<> struct codec_traits { case 8: avro::encode(e, v.get_NetworkEvent()); break; + case 9: + avro::encode(e, v.get_ProcessFlow()); + break; } } static void decode(Decoder& d, sysflow::_SysFlow_avsc_Union__4__& v) { size_t n = d.decodeUnionIndex(); - if (n >= 9) { throw avro::Exception("Union index too big"); } + if (n >= 10) { throw avro::Exception("Union index too big"); } switch (n) { case 0: { @@ -1401,6 +1507,13 @@ template<> struct codec_traits { v.set_NetworkEvent(vv); } break; + case 9: + { + sysflow::ProcessFlow vv; + avro::decode(d, vv); + v.set_ProcessFlow(vv); + } + break; } } }; diff --git a/go/converter/constants.go b/go/converter/constants.go new file mode 100644 index 0000000..3812aa4 --- /dev/null +++ b/go/converter/constants.go @@ -0,0 +1,80 @@ +package converter + +const ( + cRec = "rec" + cHPID = "hpid" + cCreateTs = "createTS" + cOID = "oid" + cPOID = "poid" + cTs = "ts" + cState = "state" + cContID = "containerId" + cID = "id" + cTID = "tid" + cOpFlags = "opFlags" + cRet = "ret" + cProcOID = "procOID" + cFileOID = "fileOID" + cEndTs = "endTs" + cFD = "fd" + cNumWSendOps = "numWSendOps" + cNumRRecvOps = "numRRecvOps" + cNumRRecvBytes = "numRRecvBytes" + cNumWSendBytes = "numWSendBytes" + cString = "string" + + cHeader = "sysflow.entity.SFHeader" + cHdrVersion = "version" + cHdrExporter = "exporter" + cHdrIP = "ip" + + cContainer = "sysflow.entity.Container" + cContName = "name" + cContImage = "image" + cContImageID = "imageid" + cContType = "type" + cContPriv = "privileged" + + cFile = "sysflow.entity.File" + cFileResType = "restype" + cFilePath = "path" + + cProcess = "sysflow.entity.Process" + cPrcExe = "exe" + cPrcExeArgs = "exeArgs" + cPrcUID = "uid" + cPrcUserName = "userName" + cPrcGid = "gid" + cPrcGroupName = "groupName" + cPrcTty = "tty" + cPrcEntry = "entry" + + cProcessEvent = "sysflow.event.ProcessEvent" + cProcEvtArgs = "args" + + cFileEvent = "sysflow.event.FileEvent" + cFileEvtFileOID = "fileOID" + cFileEvtNewFileOID = "newFileOID" + + cFileFlow = "sysflow.flow.FileFlow" + cFileFlowOpenFlags = "openFlags" + + cNetworkFlow = "sysflow.flow.NetworkFlow" + cNetFlowSIP = "sip" + cNetFlowSPort = "sport" + cNetFlowDIP = "dip" + cNetFlowDPort = "dport" + cNetFlowProto = "proto" + + cProcessFlow = "sysflow.flow.ProcessFlow" + cNumThreadsCloned = "numThreadsCloned" + cNumThreadsExited = "numThreadsExited" + cNumCloneErrors = "numCloneErrors" + + cObjectID = "sysflow.type.OID" + cFileObjectID = "sysflow.type.FOID" + + cIPIdx = 2 + cContImageRepoIdx = 6 + cPrcEntryIdx = 12 +) diff --git a/go/converter/sfconverter.go b/go/converter/sfconverter.go new file mode 100644 index 0000000..203f06c --- /dev/null +++ b/go/converter/sfconverter.go @@ -0,0 +1,292 @@ +package converter + +import ( + "time" + + "github.com/sysflow-telemetry/sf-apis/go/sfgo" + "github.ibm.com/sysflow/goutils/logger" +) + +// SFObjectConverter converts goavro schema objects into gogen objects. +type SFObjectConverter struct { +} + +// NewSFObjectConverter creates a new object which converts avro schema objects +// into sfgo sysflow objects. +func NewSFObjectConverter() *SFObjectConverter { + return new(SFObjectConverter) +} + +func (s *SFObjectConverter) createHeader(hdr map[string]interface{}) *sfgo.SFHeader { + sfhdr := &sfgo.SFHeader{ + Version: hdr[cHdrVersion].(int64), + Exporter: hdr[cHdrExporter].(string), + } + if val, ok := hdr[cHdrIP]; ok { + sfhdr.Ip = val.(string) + } else { + sfhdr.SetDefault(cIPIdx) + } + return sfhdr +} + +func (s *SFObjectConverter) createContainer(cont map[string]interface{}) *sfgo.Container { + sfcont := &sfgo.Container{ + Id: cont[cID].(string), + Name: cont[cContName].(string), + Image: cont[cContImage].(string), + Imageid: cont[cContImageID].(string), + Privileged: cont[cContPriv].(bool), + } + ct, err := sfgo.NewContainerTypeValue(cont[cContType].(string)) + if err != nil { + logger.Warn.Println("unable to extract container type mapping from: " + cont[cContType].(string)) + } else { + sfcont.Type = ct + } + + return sfcont +} + +func (s *SFObjectConverter) getTimestamp(x interface{}) int64 { + switch x.(type) { + case int64: + return x.(int64) + case time.Time: + nsecs := int64(x.(time.Time).Nanosecond() / int(time.Millisecond)) + millisecs := x.(time.Time).Unix() % int64(time.Millisecond) + secs := x.(time.Time).Unix() / int64(time.Millisecond) + nsecs += millisecs * int64(time.Microsecond) + t := time.Unix(secs, nsecs) + return t.UnixNano() + default: + logger.Warn.Printf("unknown timestamp datatype: %T", x) + } + return 0 +} + +func (s *SFObjectConverter) mapStateObject(obj string) sfgo.SFObjectState { + state, err := sfgo.NewSFObjectStateValue(obj) + if err != nil { + logger.Warn.Println("unable to extract sysflow object state type mapping from: " + obj) + } else { + return state + } + return sfgo.SFObjectStateREUP +} + +func (s *SFObjectConverter) createFile(file map[string]interface{}) *sfgo.File { + sffile := new(sfgo.File) + sffile.State = s.mapStateObject(file[cState].(string)) + copy(sffile.Oid[:], file[cOID].([]byte)) + sffile.Ts = s.getTimestamp(file[cTs]) + sffile.Restype = file[cFileResType].(int32) + sffile.Path = file[cFilePath].(string) + if val, ok := file[cContID]; ok && val != nil { + unionString := val.(map[string]interface{}) + if v, o := unionString[cString]; o { + contID := &sfgo.UnionNullString{ + String: v.(string), + UnionType: sfgo.UnionNullStringTypeEnumString, + } + sffile.ContainerId = contID + } + } else { + sffile.ContainerId = sfgo.NewUnionNullString() + } + return sffile +} + +func (s *SFObjectConverter) createOID(oid map[string]interface{}) *sfgo.OID { + if o, ok := oid[cObjectID].(map[string]interface{}); ok { + return &sfgo.OID{ + Hpid: o[cHPID].(int64), + CreateTS: s.getTimestamp(o[cCreateTs]), + } + } + return &sfgo.OID{ + Hpid: oid[cHPID].(int64), + CreateTS: s.getTimestamp(oid[cCreateTs]), + } + +} + +func (s *SFObjectConverter) createProcess(proc map[string]interface{}) *sfgo.Process { + sfproc := new(sfgo.Process) + sfproc.State = s.mapStateObject(proc[cState].(string)) + sfproc.Oid = s.createOID(proc[cOID].(map[string]interface{})) + if val, ok := proc[cPOID]; ok && val != nil { + pproc := &sfgo.UnionNullOID{ + OID: s.createOID(val.(map[string]interface{})), + UnionType: sfgo.UnionNullOIDTypeEnumOID, + } + sfproc.Poid = pproc + } else { + sfproc.Poid = sfgo.NewUnionNullOID() + } + sfproc.Ts = s.getTimestamp(proc[cTs]) + sfproc.Exe = proc[cPrcExe].(string) + sfproc.ExeArgs = proc[cPrcExeArgs].(string) + sfproc.Uid = proc[cPrcUID].(int32) + sfproc.UserName = proc[cPrcUserName].(string) + sfproc.Gid = proc[cPrcGid].(int32) + sfproc.GroupName = proc[cPrcGroupName].(string) + sfproc.Tty = proc[cPrcTty].(bool) + if val, ok := proc[cContID]; ok && val != nil { + unionString := val.(map[string]interface{}) + if v, o := unionString[cString]; o { + contID := &sfgo.UnionNullString{ + String: v.(string), + UnionType: sfgo.UnionNullStringTypeEnumString, + } + sfproc.ContainerId = contID + } + } else { + sfproc.ContainerId = sfgo.NewUnionNullString() + } + if val, ok := proc[cPrcEntry]; ok { + sfproc.Entry = val.(bool) + } else { + sfproc.SetDefault(cPrcEntryIdx) + } + return sfproc +} + +func (s *SFObjectConverter) createProcEvent(procEvt map[string]interface{}) *sfgo.ProcessEvent { + sfprocEvt := &sfgo.ProcessEvent{ + ProcOID: s.createOID(procEvt[cProcOID].(map[string]interface{})), + Ts: s.getTimestamp(procEvt[cTs]), + Tid: procEvt[cTID].(int64), + OpFlags: procEvt[cOpFlags].(int32), + Ret: procEvt[cRet].(int32), + } + if val, ok := procEvt[cProcEvtArgs].([]interface{}); ok { + for _, arg := range val { + sfprocEvt.Args = append(sfprocEvt.Args, arg.(string)) + } + } + + return sfprocEvt +} + +func (s *SFObjectConverter) createFileEvent(fileEvt map[string]interface{}) *sfgo.FileEvent { + sffileEvt := new(sfgo.FileEvent) + + sffileEvt.ProcOID = s.createOID(fileEvt[cProcOID].(map[string]interface{})) + sffileEvt.Ts = s.getTimestamp(fileEvt[cTs]) + sffileEvt.Tid = fileEvt[cTID].(int64) + sffileEvt.OpFlags = fileEvt[cOpFlags].(int32) + copy(sffileEvt.FileOID[:], fileEvt[cFileEvtFileOID].([]byte)) + sffileEvt.Ret = fileEvt[cRet].(int32) + if val, ok := fileEvt[cFileEvtNewFileOID]; ok && val != nil { + foid := val.(map[string]interface{}) + if o, ok := foid[cFileObjectID].([]byte); ok { + newFOID := &sfgo.UnionNullFOID{ + UnionType: sfgo.UnionNullFOIDTypeEnumFOID, + } + copy(newFOID.FOID[:], o) + sffileEvt.NewFileOID = newFOID + } + } else { + sffileEvt.NewFileOID = sfgo.NewUnionNullFOID() + } + + return sffileEvt +} + +func (s *SFObjectConverter) createFileFlow(fileFlow map[string]interface{}) *sfgo.FileFlow { + sffileFlow := &sfgo.FileFlow{ + ProcOID: s.createOID(fileFlow[cProcOID].(map[string]interface{})), + Ts: s.getTimestamp(fileFlow[cTs]), + Tid: fileFlow[cTID].(int64), + OpFlags: fileFlow[cOpFlags].(int32), + OpenFlags: fileFlow[cFileFlowOpenFlags].(int32), + EndTs: s.getTimestamp(fileFlow[cEndTs]), + Fd: fileFlow[cFD].(int32), + NumRRecvOps: fileFlow[cNumRRecvOps].(int64), + NumWSendOps: fileFlow[cNumWSendOps].(int64), + NumRRecvBytes: fileFlow[cNumRRecvBytes].(int64), + NumWSendBytes: fileFlow[cNumWSendBytes].(int64), + } + copy(sffileFlow.FileOID[:], fileFlow[cFileOID].([]byte)) + return sffileFlow +} + +func (s *SFObjectConverter) createProcFlow(procFlow map[string]interface{}) *sfgo.ProcessFlow { + sfprocFlow := &sfgo.ProcessFlow{ + ProcOID: s.createOID(procFlow[cProcOID].(map[string]interface{})), + Ts: s.getTimestamp(procFlow[cTs]), + NumThreadsCloned: procFlow[cNumThreadsCloned].(int64), + OpFlags: procFlow[cOpFlags].(int32), + EndTs: s.getTimestamp(procFlow[cEndTs]), + NumThreadsExited: procFlow[cNumThreadsExited].(int64), + NumCloneErrors: procFlow[cNumCloneErrors].(int64), + } + return sfprocFlow +} + +func (s *SFObjectConverter) createNetFlow(netFlow map[string]interface{}) *sfgo.NetworkFlow { + sfnetFlow := &sfgo.NetworkFlow{ + ProcOID: s.createOID(netFlow[cProcOID].(map[string]interface{})), + Ts: s.getTimestamp(netFlow[cTs]), + Tid: netFlow[cTID].(int64), + OpFlags: netFlow[cOpFlags].(int32), + EndTs: s.getTimestamp(netFlow[cEndTs]), + Fd: netFlow[cFD].(int32), + Sip: netFlow[cNetFlowSIP].(int32), + Sport: netFlow[cNetFlowSPort].(int32), + Dip: netFlow[cNetFlowDIP].(int32), + Dport: netFlow[cNetFlowDPort].(int32), + Proto: netFlow[cNetFlowProto].(int32), + NumRRecvOps: netFlow[cNumRRecvOps].(int64), + NumWSendOps: netFlow[cNumWSendOps].(int64), + NumRRecvBytes: netFlow[cNumRRecvBytes].(int64), + NumWSendBytes: netFlow[cNumWSendBytes].(int64), + } + return sfnetFlow +} + +// ConvertToSysFlow takes a datum from an OCFReader.Read() function and converts it +// into an sfgo.SysFlow object. +func (s *SFObjectConverter) ConvertToSysFlow(datum interface{}) *sfgo.SysFlow { + record := datum.(map[string]interface{}) + rec := record[cRec].(map[string]interface{}) + sFlow := sfgo.NewSysFlow() + sFlow.Rec = sfgo.NewUnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow() + for key, val := range rec { + obj := val.(map[string]interface{}) + switch key { + case cHeader: + sFlow.Rec.SFHeader = s.createHeader(obj) + sFlow.Rec.UnionType = sfgo.SF_HEADER + case cContainer: + sFlow.Rec.Container = s.createContainer(obj) + sFlow.Rec.UnionType = sfgo.SF_CONT + case cProcess: + sFlow.Rec.Process = s.createProcess(obj) + sFlow.Rec.UnionType = sfgo.SF_PROCESS + case cFile: + sFlow.Rec.File = s.createFile(obj) + sFlow.Rec.UnionType = sfgo.SF_FILE + case cProcessEvent: + sFlow.Rec.ProcessEvent = s.createProcEvent(obj) + sFlow.Rec.UnionType = sfgo.SF_PROC_EVT + case cFileEvent: + sFlow.Rec.FileEvent = s.createFileEvent(obj) + sFlow.Rec.UnionType = sfgo.SF_FILE_EVT + case cFileFlow: + sFlow.Rec.FileFlow = s.createFileFlow(obj) + sFlow.Rec.UnionType = sfgo.SF_FILE_FLOW + case cNetworkFlow: + sFlow.Rec.NetworkFlow = s.createNetFlow(obj) + sFlow.Rec.UnionType = sfgo.SF_NET_FLOW + case cProcessFlow: + sFlow.Rec.ProcessFlow = s.createProcFlow(obj) + sFlow.Rec.UnionType = sfgo.SF_PROC_FLOW + default: + logger.Error.Printf("Type: %s is currently not handled by the processor.\n", key) + + } + } + return sFlow +} diff --git a/go/generateClasses.sh b/go/generateClasses.sh new file mode 100755 index 0000000..cbcc59d --- /dev/null +++ b/go/generateClasses.sh @@ -0,0 +1,2 @@ +#!/bin/bash +gogen-avro --package=sfgo sfgo ../avro/avsc/SysFlow.avsc diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000..ad1c7ab --- /dev/null +++ b/go/go.mod @@ -0,0 +1,8 @@ +module github.com/sysflow-telemetry/sf-apis/go + +go 1.14 + +require ( + github.com/actgardner/gogen-avro v6.5.0+incompatible + github.com/actgardner/gogen-avro/v7 v7.1.1 +) diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 0000000..caa4479 --- /dev/null +++ b/go/go.sum @@ -0,0 +1,18 @@ +github.com/actgardner/gogen-avro v1.0.0 h1:LXc8axVqfHYyCTx7C/DWa9dlGDkdGhRaNCWyZoXdCqc= +github.com/actgardner/gogen-avro v6.5.0+incompatible h1:P73NiZR/S0lBWQDkK6mbvdgBXRc6e0/AaaSTqu/AvLI= +github.com/actgardner/gogen-avro v6.5.0+incompatible/go.mod h1:N2PzqZtS+5w9xxGp2daeykhWdTL0lBiRhbbvkVj4Yd8= +github.com/actgardner/gogen-avro v7.1.1+incompatible h1:tY/Kho6JroOQBszBAo/2T2obvpCsqmEb7wR3WP3eCiw= +github.com/actgardner/gogen-avro/v7 v7.1.1 h1:fAKfqQNIDIXq4Pwop3Fqu+0Tym5PuAX/cMVbdEIuVdM= +github.com/actgardner/gogen-avro/v7 v7.1.1/go.mod h1:DALbHv5zAeoz7KJ/fPAvl+d8Ixcy6x8Fjo+PO0YM8mU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= +github.com/linkedin/goavro/v2 v2.9.7/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/sysflow-telemetry/sf-apis v0.0.0-20200316230117-c2ce7cfdefde h1:d9A5xlLwJfyrMyd4VSxaKRHup/H6hek1+hAmI7mbwb8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/plugins/handler.go b/go/plugins/handler.go new file mode 100644 index 0000000..4df8e60 --- /dev/null +++ b/go/plugins/handler.go @@ -0,0 +1,27 @@ +package plugins + +import ( + "github.com/sysflow-telemetry/sf-apis/go/sfgo" +) + +// FlatChannel defines a channel for flat records. +type FlatChannel struct { + In chan *sfgo.FlatRecord +} + +// SFHandler defines the SysFlow handler interface. +type SFHandler interface { + IsEntityEnabled() bool + HandleHeader(hdr *sfgo.SFHeader) error + HandleContainer(hdr *sfgo.SFHeader, cont *sfgo.Container) error + HandleProcess(hdr *sfgo.SFHeader, cont *sfgo.Container, proc *sfgo.Process) error + HandleFile(hdr *sfgo.SFHeader, cont *sfgo.Container, file *sfgo.File) error + HandleNetFlow(hdr *sfgo.SFHeader, cont *sfgo.Container, proc *sfgo.Process, nf *sfgo.NetworkFlow) error + HandleFileFlow(hdr *sfgo.SFHeader, cont *sfgo.Container, proc *sfgo.Process, file *sfgo.File, ff *sfgo.FileFlow) error + HandleFileEvt(hdr *sfgo.SFHeader, cont *sfgo.Container, proc *sfgo.Process, file1 *sfgo.File, file2 *sfgo.File, fe *sfgo.FileEvent) error + HandleProcEvt(hdr *sfgo.SFHeader, cont *sfgo.Container, proc *sfgo.Process, pe *sfgo.ProcessEvent) error + Init(conf map[string]string) error + Register(pc SFPluginCache) + SetOutChan(ch interface{}) + Cleanup() +} diff --git a/go/plugins/plugins.go b/go/plugins/plugins.go new file mode 100644 index 0000000..e606a80 --- /dev/null +++ b/go/plugins/plugins.go @@ -0,0 +1,19 @@ +package plugins + +// Dynamic plugin function names and types for reflection. +const ( + NameFn string = "GetName" + PlugSym string = "Plugin" +) + +// SFPluginCache defines an interface for a plugin cache. +type SFPluginCache interface { + AddProcessor(name string, factory interface{}) + AddHandler(name string, factory interface{}) + AddChannel(name string, factory interface{}) +} + +// SFPluginFactory defines an abstract factory for plugins. +type SFPluginFactory interface { + Register(pc SFPluginCache) +} diff --git a/go/plugins/processor.go b/go/plugins/processor.go new file mode 100644 index 0000000..3eeb187 --- /dev/null +++ b/go/plugins/processor.go @@ -0,0 +1,22 @@ +package plugins + +import ( + "sync" + + "github.com/sysflow-telemetry/sf-apis/go/sfgo" +) + +// SFChannel defines a SysFlow channel for data transfer. +type SFChannel struct { + In chan *sfgo.SysFlow +} + +// SFProcessor defines the SysFlow processor interface. +type SFProcessor interface { + Process(record interface{}, wg *sync.WaitGroup) + Init(conf map[string]string) error + Register(pc SFPluginCache) + SetOutChan(ch interface{}) + GetName() string + Cleanup() +} diff --git a/go/sfgo/array_string.go b/go/sfgo/array_string.go new file mode 100644 index 0000000..b02df5a --- /dev/null +++ b/go/sfgo/array_string.go @@ -0,0 +1,54 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "io" + + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" +) + +func writeArrayString(r []string, w io.Writer) error { + err := vm.WriteLong(int64(len(r)), w) + if err != nil || len(r) == 0 { + return err + } + for _, e := range r { + err = vm.WriteString(e, w) + if err != nil { + return err + } + } + return vm.WriteLong(0, w) +} + +type ArrayStringWrapper struct { + Target *[]string +} + +func (_ *ArrayStringWrapper) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) SetInt(v int32) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) SetLong(v int64) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) SetString(v string) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) SetUnionElem(v int64) { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) Get(i int) types.Field { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *ArrayStringWrapper) Finalize() {} +func (_ *ArrayStringWrapper) SetDefault(i int) { panic("Unsupported operation") } +func (r *ArrayStringWrapper) NullField(i int) { + panic("Unsupported operation") +} + +func (r *ArrayStringWrapper) AppendArray() types.Field { + var v string + + *r.Target = append(*r.Target, v) + return &types.String{Target: &(*r.Target)[len(*r.Target)-1]} +} diff --git a/go/sfgo/constants.go b/go/sfgo/constants.go new file mode 100644 index 0000000..dbd6101 --- /dev/null +++ b/go/sfgo/constants.go @@ -0,0 +1,17 @@ +package sfgo + +type SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum +type SFObject = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow + +const ( + SF_HEADER SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumSFHeader + SF_CONT SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumContainer + SF_PROCESS SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcess + SF_FILE SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFile + SF_PROC_EVT SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessEvent + SF_PROC_FLOW SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessFlow + SF_NET_FLOW SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkFlow + SF_FILE_FLOW SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileFlow + SF_FILE_EVT SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileEvent + SF_NET_EVT SFObjectType = UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkEvent +) diff --git a/go/sfgo/container.go b/go/sfgo/container.go new file mode 100644 index 0000000..59ef073 --- /dev/null +++ b/go/sfgo/container.go @@ -0,0 +1,150 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type Container struct { + Id string `json:"id"` + + Name string `json:"name"` + + Image string `json:"image"` + + Imageid string `json:"imageid"` + + Type ContainerType `json:"type"` + + Privileged bool `json:"privileged"` +} + +const ContainerAvroCRC64Fingerprint = "B.\x90'A\xdd\u007f>" + +func NewContainer() *Container { + return &Container{} +} + +func DeserializeContainer(r io.Reader) (*Container, error) { + t := NewContainer() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeContainerFromSchema(r io.Reader, schema string) (*Container, error) { + t := NewContainer() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeContainer(r *Container, w io.Writer) error { + var err error + err = vm.WriteString(r.Id, w) + if err != nil { + return err + } + err = vm.WriteString(r.Name, w) + if err != nil { + return err + } + err = vm.WriteString(r.Image, w) + if err != nil { + return err + } + err = vm.WriteString(r.Imageid, w) + if err != nil { + return err + } + err = writeContainerType(r.Type, w) + if err != nil { + return err + } + err = vm.WriteBool(r.Privileged, w) + if err != nil { + return err + } + return err +} + +func (r *Container) Serialize(w io.Writer) error { + return writeContainer(r, w) +} + +func (r *Container) Schema() string { + return "{\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"image\",\"type\":\"string\"},{\"name\":\"imageid\",\"type\":\"string\"},{\"name\":\"type\",\"type\":{\"name\":\"ContainerType\",\"namespace\":\"sysflow.type\",\"symbols\":[\"CT_DOCKER\",\"CT_LXC\",\"CT_LIBVIRT_LXC\",\"CT_MESOS\",\"CT_RKT\",\"CT_CUSTOM\",\"CT_CRI\",\"CT_CONTAINERD\",\"CT_CRIO\",\"CT_BPM\"],\"type\":\"enum\"}},{\"name\":\"privileged\",\"type\":\"boolean\"}],\"name\":\"sysflow.entity.Container\",\"type\":\"record\"}" +} + +func (r *Container) SchemaName() string { + return "sysflow.entity.Container" +} + +func (_ *Container) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *Container) SetInt(v int32) { panic("Unsupported operation") } +func (_ *Container) SetLong(v int64) { panic("Unsupported operation") } +func (_ *Container) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *Container) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *Container) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *Container) SetString(v string) { panic("Unsupported operation") } +func (_ *Container) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *Container) Get(i int) types.Field { + switch i { + case 0: + return &types.String{Target: &r.Id} + case 1: + return &types.String{Target: &r.Name} + case 2: + return &types.String{Target: &r.Image} + case 3: + return &types.String{Target: &r.Imageid} + case 4: + return &ContainerTypeWrapper{Target: &r.Type} + case 5: + return &types.Boolean{Target: &r.Privileged} + } + panic("Unknown field index") +} + +func (r *Container) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *Container) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *Container) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *Container) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *Container) Finalize() {} + +func (_ *Container) AvroCRC64Fingerprint() []byte { + return []byte(ContainerAvroCRC64Fingerprint) +} diff --git a/go/sfgo/container_type.go b/go/sfgo/container_type.go new file mode 100644 index 0000000..7fd0a30 --- /dev/null +++ b/go/sfgo/container_type.go @@ -0,0 +1,160 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" +) + +type ContainerType int32 + +const ( + ContainerTypeCT_DOCKER ContainerType = 0 + ContainerTypeCT_LXC ContainerType = 1 + ContainerTypeCT_LIBVIRT_LXC ContainerType = 2 + ContainerTypeCT_MESOS ContainerType = 3 + ContainerTypeCT_RKT ContainerType = 4 + ContainerTypeCT_CUSTOM ContainerType = 5 + ContainerTypeCT_CRI ContainerType = 6 + ContainerTypeCT_CONTAINERD ContainerType = 7 + ContainerTypeCT_CRIO ContainerType = 8 + ContainerTypeCT_BPM ContainerType = 9 +) + +func (e ContainerType) String() string { + switch e { + case ContainerTypeCT_DOCKER: + return "CT_DOCKER" + case ContainerTypeCT_LXC: + return "CT_LXC" + case ContainerTypeCT_LIBVIRT_LXC: + return "CT_LIBVIRT_LXC" + case ContainerTypeCT_MESOS: + return "CT_MESOS" + case ContainerTypeCT_RKT: + return "CT_RKT" + case ContainerTypeCT_CUSTOM: + return "CT_CUSTOM" + case ContainerTypeCT_CRI: + return "CT_CRI" + case ContainerTypeCT_CONTAINERD: + return "CT_CONTAINERD" + case ContainerTypeCT_CRIO: + return "CT_CRIO" + case ContainerTypeCT_BPM: + return "CT_BPM" + } + return "unknown" +} + +func writeContainerType(r ContainerType, w io.Writer) error { + return vm.WriteInt(int32(r), w) +} + +func NewContainerTypeValue(raw string) (r ContainerType, err error) { + switch raw { + case "CT_DOCKER": + return ContainerTypeCT_DOCKER, nil + case "CT_LXC": + return ContainerTypeCT_LXC, nil + case "CT_LIBVIRT_LXC": + return ContainerTypeCT_LIBVIRT_LXC, nil + case "CT_MESOS": + return ContainerTypeCT_MESOS, nil + case "CT_RKT": + return ContainerTypeCT_RKT, nil + case "CT_CUSTOM": + return ContainerTypeCT_CUSTOM, nil + case "CT_CRI": + return ContainerTypeCT_CRI, nil + case "CT_CONTAINERD": + return ContainerTypeCT_CONTAINERD, nil + case "CT_CRIO": + return ContainerTypeCT_CRIO, nil + case "CT_BPM": + return ContainerTypeCT_BPM, nil + } + + return -1, fmt.Errorf("invalid value for ContainerType: '%s'", raw) +} + +func (b *ContainerType) MarshalJSON() ([]byte, error) { + return json.Marshal([]byte(b.String())) +} + +func (b *ContainerType) UnmarshalJSON(data []byte) error { + var stringVal string + err := json.Unmarshal(data, &stringVal) + if err != nil { + return err + } + val, err := NewContainerTypeValue(stringVal) + *b = val + return err +} + +type ContainerTypeWrapper struct { + Target *ContainerType +} + +func (b *ContainerTypeWrapper) SetBoolean(v bool) { + panic("Unable to assign boolean to int field") +} + +func (b *ContainerTypeWrapper) SetInt(v int32) { + *(b.Target) = ContainerType(v) +} + +func (b *ContainerTypeWrapper) SetLong(v int64) { + panic("Unable to assign long to int field") +} + +func (b *ContainerTypeWrapper) SetFloat(v float32) { + panic("Unable to assign float to int field") +} + +func (b *ContainerTypeWrapper) SetUnionElem(v int64) { + panic("Unable to assign union elem to int field") +} + +func (b *ContainerTypeWrapper) SetDouble(v float64) { + panic("Unable to assign double to int field") +} + +func (b *ContainerTypeWrapper) SetBytes(v []byte) { + panic("Unable to assign bytes to int field") +} + +func (b *ContainerTypeWrapper) SetString(v string) { + panic("Unable to assign string to int field") +} + +func (b *ContainerTypeWrapper) Get(i int) types.Field { + panic("Unable to get field from int field") +} + +func (b *ContainerTypeWrapper) SetDefault(i int) { + panic("Unable to set default on int field") +} + +func (b *ContainerTypeWrapper) AppendMap(key string) types.Field { + panic("Unable to append map key to from int field") +} + +func (b *ContainerTypeWrapper) AppendArray() types.Field { + panic("Unable to append array element to from int field") +} + +func (b *ContainerTypeWrapper) NullField(int) { + panic("Unable to null field in int field") +} + +func (b *ContainerTypeWrapper) Finalize() {} diff --git a/go/sfgo/file.go b/go/sfgo/file.go new file mode 100644 index 0000000..19c4d4b --- /dev/null +++ b/go/sfgo/file.go @@ -0,0 +1,155 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type File struct { + State SFObjectState `json:"state"` + + Oid FOID `json:"oid"` + + Ts int64 `json:"ts"` + + Restype int32 `json:"restype"` + + Path string `json:"path"` + + ContainerId *UnionNullString `json:"containerId"` +} + +const FileAvroCRC64Fingerprint = "g(EKٶ2\x1a" + +func NewFile() *File { + return &File{} +} + +func DeserializeFile(r io.Reader) (*File, error) { + t := NewFile() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeFileFromSchema(r io.Reader, schema string) (*File, error) { + t := NewFile() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeFile(r *File, w io.Writer) error { + var err error + err = writeSFObjectState(r.State, w) + if err != nil { + return err + } + err = writeFOID(r.Oid, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Restype, w) + if err != nil { + return err + } + err = vm.WriteString(r.Path, w) + if err != nil { + return err + } + err = writeUnionNullString(r.ContainerId, w) + if err != nil { + return err + } + return err +} + +func (r *File) Serialize(w io.Writer) error { + return writeFile(r, w) +} + +func (r *File) Schema() string { + return "{\"fields\":[{\"name\":\"state\",\"type\":{\"name\":\"SFObjectState\",\"namespace\":\"sysflow.type\",\"symbols\":[\"CREATED\",\"MODIFIED\",\"REUP\"],\"type\":\"enum\"}},{\"name\":\"oid\",\"type\":{\"name\":\"FOID\",\"namespace\":\"sysflow.type\",\"size\":20,\"type\":\"fixed\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"restype\",\"type\":\"int\"},{\"name\":\"path\",\"type\":\"string\"},{\"name\":\"containerId\",\"type\":[\"null\",\"string\"]}],\"name\":\"sysflow.entity.File\",\"type\":\"record\"}" +} + +func (r *File) SchemaName() string { + return "sysflow.entity.File" +} + +func (_ *File) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *File) SetInt(v int32) { panic("Unsupported operation") } +func (_ *File) SetLong(v int64) { panic("Unsupported operation") } +func (_ *File) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *File) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *File) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *File) SetString(v string) { panic("Unsupported operation") } +func (_ *File) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *File) Get(i int) types.Field { + switch i { + case 0: + return &SFObjectStateWrapper{Target: &r.State} + case 1: + return &FOIDWrapper{Target: &r.Oid} + case 2: + return &types.Long{Target: &r.Ts} + case 3: + return &types.Int{Target: &r.Restype} + case 4: + return &types.String{Target: &r.Path} + case 5: + r.ContainerId = NewUnionNullString() + + return r.ContainerId + } + panic("Unknown field index") +} + +func (r *File) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *File) NullField(i int) { + switch i { + case 5: + r.ContainerId = nil + return + } + panic("Not a nullable field index") +} + +func (_ *File) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *File) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *File) Finalize() {} + +func (_ *File) AvroCRC64Fingerprint() []byte { + return []byte(FileAvroCRC64Fingerprint) +} diff --git a/go/sfgo/file_event.go b/go/sfgo/file_event.go new file mode 100644 index 0000000..5ad55ad --- /dev/null +++ b/go/sfgo/file_event.go @@ -0,0 +1,165 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type FileEvent struct { + ProcOID *OID `json:"procOID"` + + Ts int64 `json:"ts"` + + Tid int64 `json:"tid"` + + OpFlags int32 `json:"opFlags"` + + FileOID FOID `json:"fileOID"` + + Ret int32 `json:"ret"` + + NewFileOID *UnionNullFOID `json:"newFileOID"` +} + +const FileEventAvroCRC64Fingerprint = "(̻5\x89\x16qK" + +func NewFileEvent() *FileEvent { + return &FileEvent{} +} + +func DeserializeFileEvent(r io.Reader) (*FileEvent, error) { + t := NewFileEvent() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeFileEventFromSchema(r io.Reader, schema string) (*FileEvent, error) { + t := NewFileEvent() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeFileEvent(r *FileEvent, w io.Writer) error { + var err error + err = writeOID(r.ProcOID, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Tid, w) + if err != nil { + return err + } + err = vm.WriteInt(r.OpFlags, w) + if err != nil { + return err + } + err = writeFOID(r.FileOID, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Ret, w) + if err != nil { + return err + } + err = writeUnionNullFOID(r.NewFileOID, w) + if err != nil { + return err + } + return err +} + +func (r *FileEvent) Serialize(w io.Writer) error { + return writeFileEvent(r, w) +} + +func (r *FileEvent) Schema() string { + return "{\"fields\":[{\"name\":\"procOID\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"fileOID\",\"type\":{\"name\":\"FOID\",\"namespace\":\"sysflow.type\",\"size\":20,\"type\":\"fixed\"}},{\"name\":\"ret\",\"type\":\"int\"},{\"name\":\"newFileOID\",\"type\":[\"null\",\"sysflow.type.FOID\"]}],\"name\":\"sysflow.event.FileEvent\",\"type\":\"record\"}" +} + +func (r *FileEvent) SchemaName() string { + return "sysflow.event.FileEvent" +} + +func (_ *FileEvent) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *FileEvent) SetInt(v int32) { panic("Unsupported operation") } +func (_ *FileEvent) SetLong(v int64) { panic("Unsupported operation") } +func (_ *FileEvent) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *FileEvent) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *FileEvent) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *FileEvent) SetString(v string) { panic("Unsupported operation") } +func (_ *FileEvent) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *FileEvent) Get(i int) types.Field { + switch i { + case 0: + r.ProcOID = NewOID() + + return r.ProcOID + case 1: + return &types.Long{Target: &r.Ts} + case 2: + return &types.Long{Target: &r.Tid} + case 3: + return &types.Int{Target: &r.OpFlags} + case 4: + return &FOIDWrapper{Target: &r.FileOID} + case 5: + return &types.Int{Target: &r.Ret} + case 6: + r.NewFileOID = NewUnionNullFOID() + + return r.NewFileOID + } + panic("Unknown field index") +} + +func (r *FileEvent) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *FileEvent) NullField(i int) { + switch i { + case 6: + r.NewFileOID = nil + return + } + panic("Not a nullable field index") +} + +func (_ *FileEvent) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *FileEvent) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *FileEvent) Finalize() {} + +func (_ *FileEvent) AvroCRC64Fingerprint() []byte { + return []byte(FileEventAvroCRC64Fingerprint) +} diff --git a/go/sfgo/file_flow.go b/go/sfgo/file_flow.go new file mode 100644 index 0000000..f49d9fe --- /dev/null +++ b/go/sfgo/file_flow.go @@ -0,0 +1,200 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type FileFlow struct { + ProcOID *OID `json:"procOID"` + + Ts int64 `json:"ts"` + + Tid int64 `json:"tid"` + + OpFlags int32 `json:"opFlags"` + + OpenFlags int32 `json:"openFlags"` + + EndTs int64 `json:"endTs"` + + FileOID FOID `json:"fileOID"` + + Fd int32 `json:"fd"` + + NumRRecvOps int64 `json:"numRRecvOps"` + + NumWSendOps int64 `json:"numWSendOps"` + + NumRRecvBytes int64 `json:"numRRecvBytes"` + + NumWSendBytes int64 `json:"numWSendBytes"` +} + +const FileFlowAvroCRC64Fingerprint = "\xb0\x9a\xb1-\x80\x85G\\" + +func NewFileFlow() *FileFlow { + return &FileFlow{} +} + +func DeserializeFileFlow(r io.Reader) (*FileFlow, error) { + t := NewFileFlow() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeFileFlowFromSchema(r io.Reader, schema string) (*FileFlow, error) { + t := NewFileFlow() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeFileFlow(r *FileFlow, w io.Writer) error { + var err error + err = writeOID(r.ProcOID, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Tid, w) + if err != nil { + return err + } + err = vm.WriteInt(r.OpFlags, w) + if err != nil { + return err + } + err = vm.WriteInt(r.OpenFlags, w) + if err != nil { + return err + } + err = vm.WriteLong(r.EndTs, w) + if err != nil { + return err + } + err = writeFOID(r.FileOID, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Fd, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumRRecvOps, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumWSendOps, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumRRecvBytes, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumWSendBytes, w) + if err != nil { + return err + } + return err +} + +func (r *FileFlow) Serialize(w io.Writer) error { + return writeFileFlow(r, w) +} + +func (r *FileFlow) Schema() string { + return "{\"fields\":[{\"name\":\"procOID\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"openFlags\",\"type\":\"int\"},{\"name\":\"endTs\",\"type\":\"long\"},{\"name\":\"fileOID\",\"type\":{\"name\":\"FOID\",\"namespace\":\"sysflow.type\",\"size\":20,\"type\":\"fixed\"}},{\"name\":\"fd\",\"type\":\"int\"},{\"name\":\"numRRecvOps\",\"type\":\"long\"},{\"name\":\"numWSendOps\",\"type\":\"long\"},{\"name\":\"numRRecvBytes\",\"type\":\"long\"},{\"name\":\"numWSendBytes\",\"type\":\"long\"}],\"name\":\"sysflow.flow.FileFlow\",\"type\":\"record\"}" +} + +func (r *FileFlow) SchemaName() string { + return "sysflow.flow.FileFlow" +} + +func (_ *FileFlow) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *FileFlow) SetInt(v int32) { panic("Unsupported operation") } +func (_ *FileFlow) SetLong(v int64) { panic("Unsupported operation") } +func (_ *FileFlow) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *FileFlow) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *FileFlow) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *FileFlow) SetString(v string) { panic("Unsupported operation") } +func (_ *FileFlow) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *FileFlow) Get(i int) types.Field { + switch i { + case 0: + r.ProcOID = NewOID() + + return r.ProcOID + case 1: + return &types.Long{Target: &r.Ts} + case 2: + return &types.Long{Target: &r.Tid} + case 3: + return &types.Int{Target: &r.OpFlags} + case 4: + return &types.Int{Target: &r.OpenFlags} + case 5: + return &types.Long{Target: &r.EndTs} + case 6: + return &FOIDWrapper{Target: &r.FileOID} + case 7: + return &types.Int{Target: &r.Fd} + case 8: + return &types.Long{Target: &r.NumRRecvOps} + case 9: + return &types.Long{Target: &r.NumWSendOps} + case 10: + return &types.Long{Target: &r.NumRRecvBytes} + case 11: + return &types.Long{Target: &r.NumWSendBytes} + } + panic("Unknown field index") +} + +func (r *FileFlow) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *FileFlow) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *FileFlow) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *FileFlow) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *FileFlow) Finalize() {} + +func (_ *FileFlow) AvroCRC64Fingerprint() []byte { + return []byte(FileFlowAvroCRC64Fingerprint) +} diff --git a/go/sfgo/flat_constants.go b/go/sfgo/flat_constants.go new file mode 100644 index 0000000..522c6d7 --- /dev/null +++ b/go/sfgo/flat_constants.go @@ -0,0 +1,97 @@ +// Code generate by sfindex. DO NOT EDIT. +package sfgo + +type Attribute uint32 + +const ( + ARRAY_INT Attribute = 0 + ARRAY_STR Attribute = 0 + + SF_REC_TYPE Attribute = ARRAY_INT + //Type name: SFHeader + SFHE_VERSION_INT Attribute = SF_REC_TYPE + 1 + SFHE_EXPORTER_STR Attribute = ARRAY_STR + SFHE_IP_STR Attribute = SFHE_EXPORTER_STR + 1 + + //Type name: Container + CONT_ID_STR Attribute = SFHE_IP_STR + 1 + CONT_NAME_STR Attribute = CONT_ID_STR + 1 + CONT_IMAGE_STR Attribute = CONT_NAME_STR + 1 + CONT_IMAGEID_STR Attribute = CONT_IMAGE_STR + 1 + CONT_TYPE_INT Attribute = SFHE_VERSION_INT + 1 + CONT_PRIVILEGED_INT Attribute = CONT_TYPE_INT + 1 + + //Type name: Process + PROC_STATE_INT Attribute = CONT_PRIVILEGED_INT + 1 + PROC_OID_CREATETS_INT Attribute = PROC_STATE_INT + 1 + PROC_OID_HPID_INT Attribute = PROC_OID_CREATETS_INT + 1 + PROC_POID_CREATETS_INT Attribute = PROC_OID_HPID_INT + 1 + PROC_POID_HPID_INT Attribute = PROC_POID_CREATETS_INT + 1 + PROC_TS_INT Attribute = PROC_POID_HPID_INT + 1 + PROC_EXE_STR Attribute = CONT_IMAGEID_STR + 1 + PROC_EXEARGS_STR Attribute = PROC_EXE_STR + 1 + PROC_UID_INT Attribute = PROC_TS_INT + 1 + PROC_USERNAME_STR Attribute = PROC_EXEARGS_STR + 1 + PROC_GID_INT Attribute = PROC_UID_INT + 1 + PROC_GROUPNAME_STR Attribute = PROC_USERNAME_STR + 1 + PROC_TTY_INT Attribute = PROC_GID_INT + 1 + PROC_CONTAINERID_STRING_STR Attribute = PROC_GROUPNAME_STR + 1 + PROC_ENTRY_INT Attribute = PROC_TTY_INT + 1 + + //Type name: File + FILE_STATE_INT Attribute = PROC_ENTRY_INT + 1 + FILE_TS_INT Attribute = FILE_STATE_INT + 1 + FILE_RESTYPE_INT Attribute = FILE_TS_INT + 1 + FILE_PATH_STR Attribute = PROC_CONTAINERID_STRING_STR + 1 + FILE_CONTAINERID_STRING_STR Attribute = FILE_PATH_STR + 1 + + //Type name: FileEvent + EV_FILE_TS_INT Attribute = FILE_RESTYPE_INT + 1 + EV_FILE_TID_INT Attribute = EV_FILE_TS_INT + 1 + EV_FILE_OPFLAGS_INT Attribute = EV_FILE_TID_INT + 1 + EV_FILE_RET_INT Attribute = EV_FILE_OPFLAGS_INT + 1 + + //Type name: File number 2 + SEC_FILE_STATE_INT Attribute = EV_FILE_RET_INT + 1 + SEC_FILE_TS_INT Attribute = SEC_FILE_STATE_INT + 1 + SEC_FILE_RESTYPE_INT Attribute = SEC_FILE_TS_INT + 1 + SEC_FILE_PATH_STR Attribute = FILE_CONTAINERID_STRING_STR + 1 + SEC_FILE_CONTAINERID_STRING_STR Attribute = SEC_FILE_PATH_STR + 1 + + //Type name: FileFlow + FL_FILE_TS_INT Attribute = FILE_RESTYPE_INT + 1 + FL_FILE_TID_INT Attribute = FL_FILE_TS_INT + 1 + FL_FILE_OPFLAGS_INT Attribute = FL_FILE_TID_INT + 1 + FL_FILE_ENDTS_INT Attribute = FL_FILE_OPFLAGS_INT + 1 + FL_FILE_FD_INT Attribute = FL_FILE_ENDTS_INT + 1 + FL_FILE_NUMRRECVOPS_INT Attribute = FL_FILE_FD_INT + 1 + FL_FILE_NUMWSENDOPS_INT Attribute = FL_FILE_NUMRRECVOPS_INT + 1 + FL_FILE_NUMRRECVBYTES_INT Attribute = FL_FILE_NUMWSENDOPS_INT + 1 + FL_FILE_NUMWSENDBYTES_INT Attribute = FL_FILE_NUMRRECVBYTES_INT + 1 + FL_FILE_OPENFLAGS_INT Attribute = FL_FILE_NUMWSENDBYTES_INT + 1 + + //Type name: NetworkFlow + FL_NETW_TS_INT Attribute = FILE_RESTYPE_INT + 1 + FL_NETW_TID_INT Attribute = FL_NETW_TS_INT + 1 + FL_NETW_OPFLAGS_INT Attribute = FL_NETW_TID_INT + 1 + FL_NETW_ENDTS_INT Attribute = FL_NETW_OPFLAGS_INT + 1 + FL_NETW_FD_INT Attribute = FL_NETW_ENDTS_INT + 1 + FL_NETW_NUMRRECVOPS_INT Attribute = FL_NETW_FD_INT + 1 + FL_NETW_NUMWSENDOPS_INT Attribute = FL_NETW_NUMRRECVOPS_INT + 1 + FL_NETW_NUMRRECVBYTES_INT Attribute = FL_NETW_NUMWSENDOPS_INT + 1 + FL_NETW_NUMWSENDBYTES_INT Attribute = FL_NETW_NUMRRECVBYTES_INT + 1 + FL_NETW_SIP_INT Attribute = FL_NETW_NUMWSENDBYTES_INT + 1 + FL_NETW_SPORT_INT Attribute = FL_NETW_SIP_INT + 1 + FL_NETW_DIP_INT Attribute = FL_NETW_SPORT_INT + 1 + FL_NETW_DPORT_INT Attribute = FL_NETW_DIP_INT + 1 + FL_NETW_PROTO_INT Attribute = FL_NETW_DPORT_INT + 1 + + //Type name: ProcessEvent + EV_PROC_TS_INT Attribute = FILE_RESTYPE_INT + 1 + EV_PROC_TID_INT Attribute = EV_PROC_TS_INT + 1 + EV_PROC_OPFLAGS_INT Attribute = EV_PROC_TID_INT + 1 + EV_PROC_RET_INT Attribute = EV_PROC_OPFLAGS_INT + 1 + + INT_ARRAY_SIZE Attribute = 30 + 1 + STR_ARRAY_SIZE Attribute = 14 + 1 +) diff --git a/go/sfgo/flatrecord.go b/go/sfgo/flatrecord.go new file mode 100644 index 0000000..bc094b9 --- /dev/null +++ b/go/sfgo/flatrecord.go @@ -0,0 +1,61 @@ +package sfgo + +const ( + TS_INT Attribute = EV_FILE_TS_INT + TID_INT Attribute = EV_FILE_TID_INT + OPFLAGS_INT Attribute = EV_FILE_OPFLAGS_INT + RET_INT Attribute = EV_FILE_RET_INT + + ENDTS_INT Attribute = FL_FILE_ENDTS_INT + FD_INT Attribute = FL_FILE_FD_INT + NUMRRECVOPS_INT Attribute = FL_FILE_NUMRRECVOPS_INT + NUMWSENDOPS_INT Attribute = FL_FILE_NUMWSENDOPS_INT + NUMRRECVBYTES_INT Attribute = FL_FILE_NUMRRECVBYTES_INT + NUMWSENDBYTES_INT Attribute = FL_FILE_NUMWSENDBYTES_INT + + HEADER int64 = 0 + CONT int64 = 1 + PROC int64 = 2 + FILE int64 = 3 + PROC_EVT int64 = 4 + NET_FLOW int64 = 5 + FILE_FLOW int64 = 6 + FILE_EVT int64 = 7 +) + +// sftypes is used to obtain zero values for types used during flattening. +type sftypes struct { + Int64 int64 + String string +} + +// Zeros is a zero-initialized struct used to obtain zero values for types used during flattening. +var Zeros = sftypes{} + +// FlatRecord represents a flattened SysFlow record +type FlatRecord struct { + Ints [INT_ARRAY_SIZE]int64 + Strs [STR_ARRAY_SIZE]string +} + +// type FlatRecordPool struct { +// pool *sync.Pool +// } + +// func NewFlatRecordPool() *FlatRecordPool { +// p := new(FlatRecordPool) +// p.pool = &sync.Pool{ +// New: func() interface{} { +// return new(FlatRecord) +// }, +// } +// return p +// } + +// func (f *FlatRecordPool) Get() *FlatRecord { +// return f.pool.Get().(*FlatRecord) +// } + +// func (f *FlatRecordPool) Put(fr *FlatRecord) { +// f.pool.Put(fr) +// } diff --git a/go/sfgo/foid.go b/go/sfgo/foid.go new file mode 100644 index 0000000..846d074 --- /dev/null +++ b/go/sfgo/foid.go @@ -0,0 +1,39 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +func writeFOID(r FOID, w io.Writer) error { + _, err := w.Write(r[:]) + return err +} + +type FOIDWrapper struct { + Target *FOID +} + +type FOID [20]byte + +func (_ *FOIDWrapper) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *FOIDWrapper) SetInt(v int32) { panic("Unsupported operation") } +func (_ *FOIDWrapper) SetLong(v int64) { panic("Unsupported operation") } +func (_ *FOIDWrapper) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *FOIDWrapper) SetDouble(v float64) { panic("Unsupported operation") } +func (r *FOIDWrapper) SetBytes(v []byte) { + copy((*r.Target)[:], v) +} +func (_ *FOIDWrapper) SetString(v string) { panic("Unsupported operation") } +func (_ *FOIDWrapper) SetUnionElem(v int64) { panic("Unsupported operation") } +func (_ *FOIDWrapper) Get(i int) types.Field { panic("Unsupported operation") } +func (_ *FOIDWrapper) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *FOIDWrapper) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *FOIDWrapper) NullField(int) { panic("Unsupported operation") } +func (_ *FOIDWrapper) Finalize() {} +func (_ *FOIDWrapper) SetDefault(i int) { panic("Unsupported operation") } diff --git a/go/sfgo/network_event.go b/go/sfgo/network_event.go new file mode 100644 index 0000000..fa1c3e9 --- /dev/null +++ b/go/sfgo/network_event.go @@ -0,0 +1,184 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type NetworkEvent struct { + ProcOID *OID `json:"procOID"` + + Ts int64 `json:"ts"` + + Tid int64 `json:"tid"` + + OpFlags int32 `json:"opFlags"` + + Sip int32 `json:"sip"` + + Sport int32 `json:"sport"` + + Dip int32 `json:"dip"` + + Dport int32 `json:"dport"` + + Proto int32 `json:"proto"` + + Ret int32 `json:"ret"` +} + +const NetworkEventAvroCRC64Fingerprint = "\xd9g\x01ј\xe1\xc2\xd0" + +func NewNetworkEvent() *NetworkEvent { + return &NetworkEvent{} +} + +func DeserializeNetworkEvent(r io.Reader) (*NetworkEvent, error) { + t := NewNetworkEvent() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeNetworkEventFromSchema(r io.Reader, schema string) (*NetworkEvent, error) { + t := NewNetworkEvent() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeNetworkEvent(r *NetworkEvent, w io.Writer) error { + var err error + err = writeOID(r.ProcOID, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Tid, w) + if err != nil { + return err + } + err = vm.WriteInt(r.OpFlags, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Sip, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Sport, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Dip, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Dport, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Proto, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Ret, w) + if err != nil { + return err + } + return err +} + +func (r *NetworkEvent) Serialize(w io.Writer) error { + return writeNetworkEvent(r, w) +} + +func (r *NetworkEvent) Schema() string { + return "{\"fields\":[{\"name\":\"procOID\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"sip\",\"type\":\"int\"},{\"name\":\"sport\",\"type\":\"int\"},{\"name\":\"dip\",\"type\":\"int\"},{\"name\":\"dport\",\"type\":\"int\"},{\"name\":\"proto\",\"type\":\"int\"},{\"name\":\"ret\",\"type\":\"int\"}],\"name\":\"sysflow.event.NetworkEvent\",\"type\":\"record\"}" +} + +func (r *NetworkEvent) SchemaName() string { + return "sysflow.event.NetworkEvent" +} + +func (_ *NetworkEvent) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *NetworkEvent) SetInt(v int32) { panic("Unsupported operation") } +func (_ *NetworkEvent) SetLong(v int64) { panic("Unsupported operation") } +func (_ *NetworkEvent) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *NetworkEvent) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *NetworkEvent) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *NetworkEvent) SetString(v string) { panic("Unsupported operation") } +func (_ *NetworkEvent) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *NetworkEvent) Get(i int) types.Field { + switch i { + case 0: + r.ProcOID = NewOID() + + return r.ProcOID + case 1: + return &types.Long{Target: &r.Ts} + case 2: + return &types.Long{Target: &r.Tid} + case 3: + return &types.Int{Target: &r.OpFlags} + case 4: + return &types.Int{Target: &r.Sip} + case 5: + return &types.Int{Target: &r.Sport} + case 6: + return &types.Int{Target: &r.Dip} + case 7: + return &types.Int{Target: &r.Dport} + case 8: + return &types.Int{Target: &r.Proto} + case 9: + return &types.Int{Target: &r.Ret} + } + panic("Unknown field index") +} + +func (r *NetworkEvent) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *NetworkEvent) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *NetworkEvent) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *NetworkEvent) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *NetworkEvent) Finalize() {} + +func (_ *NetworkEvent) AvroCRC64Fingerprint() []byte { + return []byte(NetworkEventAvroCRC64Fingerprint) +} diff --git a/go/sfgo/network_flow.go b/go/sfgo/network_flow.go new file mode 100644 index 0000000..53c4a5d --- /dev/null +++ b/go/sfgo/network_flow.go @@ -0,0 +1,224 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type NetworkFlow struct { + ProcOID *OID `json:"procOID"` + + Ts int64 `json:"ts"` + + Tid int64 `json:"tid"` + + OpFlags int32 `json:"opFlags"` + + EndTs int64 `json:"endTs"` + + Sip int32 `json:"sip"` + + Sport int32 `json:"sport"` + + Dip int32 `json:"dip"` + + Dport int32 `json:"dport"` + + Proto int32 `json:"proto"` + + Fd int32 `json:"fd"` + + NumRRecvOps int64 `json:"numRRecvOps"` + + NumWSendOps int64 `json:"numWSendOps"` + + NumRRecvBytes int64 `json:"numRRecvBytes"` + + NumWSendBytes int64 `json:"numWSendBytes"` +} + +const NetworkFlowAvroCRC64Fingerprint = "K\xd5\x14\xceg\xe2 \xd3" + +func NewNetworkFlow() *NetworkFlow { + return &NetworkFlow{} +} + +func DeserializeNetworkFlow(r io.Reader) (*NetworkFlow, error) { + t := NewNetworkFlow() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeNetworkFlowFromSchema(r io.Reader, schema string) (*NetworkFlow, error) { + t := NewNetworkFlow() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeNetworkFlow(r *NetworkFlow, w io.Writer) error { + var err error + err = writeOID(r.ProcOID, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Tid, w) + if err != nil { + return err + } + err = vm.WriteInt(r.OpFlags, w) + if err != nil { + return err + } + err = vm.WriteLong(r.EndTs, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Sip, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Sport, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Dip, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Dport, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Proto, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Fd, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumRRecvOps, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumWSendOps, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumRRecvBytes, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumWSendBytes, w) + if err != nil { + return err + } + return err +} + +func (r *NetworkFlow) Serialize(w io.Writer) error { + return writeNetworkFlow(r, w) +} + +func (r *NetworkFlow) Schema() string { + return "{\"fields\":[{\"name\":\"procOID\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"endTs\",\"type\":\"long\"},{\"name\":\"sip\",\"type\":\"int\"},{\"name\":\"sport\",\"type\":\"int\"},{\"name\":\"dip\",\"type\":\"int\"},{\"name\":\"dport\",\"type\":\"int\"},{\"name\":\"proto\",\"type\":\"int\"},{\"name\":\"fd\",\"type\":\"int\"},{\"name\":\"numRRecvOps\",\"type\":\"long\"},{\"name\":\"numWSendOps\",\"type\":\"long\"},{\"name\":\"numRRecvBytes\",\"type\":\"long\"},{\"name\":\"numWSendBytes\",\"type\":\"long\"}],\"name\":\"sysflow.flow.NetworkFlow\",\"type\":\"record\"}" +} + +func (r *NetworkFlow) SchemaName() string { + return "sysflow.flow.NetworkFlow" +} + +func (_ *NetworkFlow) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *NetworkFlow) SetInt(v int32) { panic("Unsupported operation") } +func (_ *NetworkFlow) SetLong(v int64) { panic("Unsupported operation") } +func (_ *NetworkFlow) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *NetworkFlow) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *NetworkFlow) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *NetworkFlow) SetString(v string) { panic("Unsupported operation") } +func (_ *NetworkFlow) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *NetworkFlow) Get(i int) types.Field { + switch i { + case 0: + r.ProcOID = NewOID() + + return r.ProcOID + case 1: + return &types.Long{Target: &r.Ts} + case 2: + return &types.Long{Target: &r.Tid} + case 3: + return &types.Int{Target: &r.OpFlags} + case 4: + return &types.Long{Target: &r.EndTs} + case 5: + return &types.Int{Target: &r.Sip} + case 6: + return &types.Int{Target: &r.Sport} + case 7: + return &types.Int{Target: &r.Dip} + case 8: + return &types.Int{Target: &r.Dport} + case 9: + return &types.Int{Target: &r.Proto} + case 10: + return &types.Int{Target: &r.Fd} + case 11: + return &types.Long{Target: &r.NumRRecvOps} + case 12: + return &types.Long{Target: &r.NumWSendOps} + case 13: + return &types.Long{Target: &r.NumRRecvBytes} + case 14: + return &types.Long{Target: &r.NumWSendBytes} + } + panic("Unknown field index") +} + +func (r *NetworkFlow) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *NetworkFlow) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *NetworkFlow) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *NetworkFlow) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *NetworkFlow) Finalize() {} + +func (_ *NetworkFlow) AvroCRC64Fingerprint() []byte { + return []byte(NetworkFlowAvroCRC64Fingerprint) +} diff --git a/go/sfgo/oid.go b/go/sfgo/oid.go new file mode 100644 index 0000000..2d7a23a --- /dev/null +++ b/go/sfgo/oid.go @@ -0,0 +1,118 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type OID struct { + CreateTS int64 `json:"createTS"` + + Hpid int64 `json:"hpid"` +} + +const OIDAvroCRC64Fingerprint = "\x98ׅf\x87\xe7\u0602" + +func NewOID() *OID { + return &OID{} +} + +func DeserializeOID(r io.Reader) (*OID, error) { + t := NewOID() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeOIDFromSchema(r io.Reader, schema string) (*OID, error) { + t := NewOID() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeOID(r *OID, w io.Writer) error { + var err error + err = vm.WriteLong(r.CreateTS, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Hpid, w) + if err != nil { + return err + } + return err +} + +func (r *OID) Serialize(w io.Writer) error { + return writeOID(r, w) +} + +func (r *OID) Schema() string { + return "{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"sysflow.type.OID\",\"type\":\"record\"}" +} + +func (r *OID) SchemaName() string { + return "sysflow.type.OID" +} + +func (_ *OID) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *OID) SetInt(v int32) { panic("Unsupported operation") } +func (_ *OID) SetLong(v int64) { panic("Unsupported operation") } +func (_ *OID) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *OID) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *OID) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *OID) SetString(v string) { panic("Unsupported operation") } +func (_ *OID) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *OID) Get(i int) types.Field { + switch i { + case 0: + return &types.Long{Target: &r.CreateTS} + case 1: + return &types.Long{Target: &r.Hpid} + } + panic("Unknown field index") +} + +func (r *OID) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *OID) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *OID) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *OID) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *OID) Finalize() {} + +func (_ *OID) AvroCRC64Fingerprint() []byte { + return []byte(OIDAvroCRC64Fingerprint) +} diff --git a/go/sfgo/op_flags.go b/go/sfgo/op_flags.go new file mode 100644 index 0000000..9888179 --- /dev/null +++ b/go/sfgo/op_flags.go @@ -0,0 +1,25 @@ +package sfgo + +const ( + OP_CLONE = (1 << 0) + OP_EXEC = (1 << 1) + OP_EXIT = (1 << 2) + OP_SETUID = (1 << 3) + OP_SETNS = (1 << 4) + OP_ACCEPT = (1 << 5) + OP_CONNECT = (1 << 6) + OP_OPEN = (1 << 7) + OP_READ_RECV = (1 << 8) + OP_WRITE_SEND = (1 << 9) + OP_CLOSE = (1 << 10) + OP_TRUNCATE = (1 << 11) + OP_SHUTDOWN = (1 << 12) + OP_MMAP = (1 << 13) + OP_DIGEST = (1 << 14) + OP_MKDIR = (1 << 15) + OP_RMDIR = (1 << 16) + OP_LINK = (1 << 17) + OP_UNLINK = (1 << 18) + OP_SYMLINK = (1 << 19) + OP_RENAME = (1 << 20) +) diff --git a/go/sfgo/open_flags.go b/go/sfgo/open_flags.go new file mode 100644 index 0000000..7207a13 --- /dev/null +++ b/go/sfgo/open_flags.go @@ -0,0 +1,20 @@ +package sfgo + +// Open Flags +const ( + O_NONE = (0) + O_RDONLY = (1 << 0) // Open for reading only + O_WRONLY = (1 << 1) // Open for writing only + O_RDWR = (O_RDONLY | O_WRONLY) // Open for reading and writing + O_CREAT = (1 << 2) // Create a new file if it doesn't exist. + O_APPEND = (1 << 3) // If set, the file offset shall be set to the end of the file prior to each write. + O_DSYNC = (1 << 4) + O_EXCL = (1 << 5) + O_NONBLOCK = (1 << 6) + O_SYNC = (1 << 7) + O_TRUNC = (1 << 8) + O_DIRECT = (1 << 9) + O_DIRECTORY = (1 << 10) + O_LARGEFILE = (1 << 11) + O_CLOEXEC = (1 << 12) +) diff --git a/go/sfgo/process.go b/go/sfgo/process.go new file mode 100644 index 0000000..b535110 --- /dev/null +++ b/go/sfgo/process.go @@ -0,0 +1,221 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type Process struct { + State SFObjectState `json:"state"` + + Oid *OID `json:"oid"` + + Poid *UnionNullOID `json:"poid"` + + Ts int64 `json:"ts"` + + Exe string `json:"exe"` + + ExeArgs string `json:"exeArgs"` + + Uid int32 `json:"uid"` + + UserName string `json:"userName"` + + Gid int32 `json:"gid"` + + GroupName string `json:"groupName"` + + Tty bool `json:"tty"` + + ContainerId *UnionNullString `json:"containerId"` + + Entry bool `json:"entry"` +} + +const ProcessAvroCRC64Fingerprint = "\x12~\xc3ίqO|" + +func NewProcess() *Process { + return &Process{} +} + +func DeserializeProcess(r io.Reader) (*Process, error) { + t := NewProcess() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeProcessFromSchema(r io.Reader, schema string) (*Process, error) { + t := NewProcess() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeProcess(r *Process, w io.Writer) error { + var err error + err = writeSFObjectState(r.State, w) + if err != nil { + return err + } + err = writeOID(r.Oid, w) + if err != nil { + return err + } + err = writeUnionNullOID(r.Poid, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteString(r.Exe, w) + if err != nil { + return err + } + err = vm.WriteString(r.ExeArgs, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Uid, w) + if err != nil { + return err + } + err = vm.WriteString(r.UserName, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Gid, w) + if err != nil { + return err + } + err = vm.WriteString(r.GroupName, w) + if err != nil { + return err + } + err = vm.WriteBool(r.Tty, w) + if err != nil { + return err + } + err = writeUnionNullString(r.ContainerId, w) + if err != nil { + return err + } + err = vm.WriteBool(r.Entry, w) + if err != nil { + return err + } + return err +} + +func (r *Process) Serialize(w io.Writer) error { + return writeProcess(r, w) +} + +func (r *Process) Schema() string { + return "{\"fields\":[{\"name\":\"state\",\"type\":{\"name\":\"SFObjectState\",\"namespace\":\"sysflow.type\",\"symbols\":[\"CREATED\",\"MODIFIED\",\"REUP\"],\"type\":\"enum\"}},{\"name\":\"oid\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"poid\",\"type\":[\"null\",\"sysflow.type.OID\"]},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"exe\",\"type\":\"string\"},{\"name\":\"exeArgs\",\"type\":\"string\"},{\"name\":\"uid\",\"type\":\"int\"},{\"name\":\"userName\",\"type\":\"string\"},{\"name\":\"gid\",\"type\":\"int\"},{\"name\":\"groupName\",\"type\":\"string\"},{\"name\":\"tty\",\"type\":\"boolean\"},{\"name\":\"containerId\",\"type\":[\"null\",\"string\"]},{\"default\":false,\"name\":\"entry\",\"type\":\"boolean\"}],\"name\":\"sysflow.entity.Process\",\"type\":\"record\"}" +} + +func (r *Process) SchemaName() string { + return "sysflow.entity.Process" +} + +func (_ *Process) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *Process) SetInt(v int32) { panic("Unsupported operation") } +func (_ *Process) SetLong(v int64) { panic("Unsupported operation") } +func (_ *Process) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *Process) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *Process) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *Process) SetString(v string) { panic("Unsupported operation") } +func (_ *Process) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *Process) Get(i int) types.Field { + switch i { + case 0: + return &SFObjectStateWrapper{Target: &r.State} + case 1: + r.Oid = NewOID() + + return r.Oid + case 2: + r.Poid = NewUnionNullOID() + + return r.Poid + case 3: + return &types.Long{Target: &r.Ts} + case 4: + return &types.String{Target: &r.Exe} + case 5: + return &types.String{Target: &r.ExeArgs} + case 6: + return &types.Int{Target: &r.Uid} + case 7: + return &types.String{Target: &r.UserName} + case 8: + return &types.Int{Target: &r.Gid} + case 9: + return &types.String{Target: &r.GroupName} + case 10: + return &types.Boolean{Target: &r.Tty} + case 11: + r.ContainerId = NewUnionNullString() + + return r.ContainerId + case 12: + return &types.Boolean{Target: &r.Entry} + } + panic("Unknown field index") +} + +func (r *Process) SetDefault(i int) { + switch i { + case 12: + r.Entry = false + return + } + panic("Unknown field index") +} + +func (r *Process) NullField(i int) { + switch i { + case 2: + r.Poid = nil + return + case 11: + r.ContainerId = nil + return + } + panic("Not a nullable field index") +} + +func (_ *Process) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *Process) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *Process) Finalize() {} + +func (_ *Process) AvroCRC64Fingerprint() []byte { + return []byte(ProcessAvroCRC64Fingerprint) +} diff --git a/go/sfgo/process_event.go b/go/sfgo/process_event.go new file mode 100644 index 0000000..cee430b --- /dev/null +++ b/go/sfgo/process_event.go @@ -0,0 +1,154 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type ProcessEvent struct { + ProcOID *OID `json:"procOID"` + + Ts int64 `json:"ts"` + + Tid int64 `json:"tid"` + + OpFlags int32 `json:"opFlags"` + + Args []string `json:"args"` + + Ret int32 `json:"ret"` +} + +const ProcessEventAvroCRC64Fingerprint = "\xa3\x98\xb0\xfe\x1do+\xd9" + +func NewProcessEvent() *ProcessEvent { + return &ProcessEvent{} +} + +func DeserializeProcessEvent(r io.Reader) (*ProcessEvent, error) { + t := NewProcessEvent() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeProcessEventFromSchema(r io.Reader, schema string) (*ProcessEvent, error) { + t := NewProcessEvent() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeProcessEvent(r *ProcessEvent, w io.Writer) error { + var err error + err = writeOID(r.ProcOID, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Tid, w) + if err != nil { + return err + } + err = vm.WriteInt(r.OpFlags, w) + if err != nil { + return err + } + err = writeArrayString(r.Args, w) + if err != nil { + return err + } + err = vm.WriteInt(r.Ret, w) + if err != nil { + return err + } + return err +} + +func (r *ProcessEvent) Serialize(w io.Writer) error { + return writeProcessEvent(r, w) +} + +func (r *ProcessEvent) Schema() string { + return "{\"fields\":[{\"name\":\"procOID\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"items\":\"string\",\"type\":\"array\"}},{\"name\":\"ret\",\"type\":\"int\"}],\"name\":\"sysflow.event.ProcessEvent\",\"type\":\"record\"}" +} + +func (r *ProcessEvent) SchemaName() string { + return "sysflow.event.ProcessEvent" +} + +func (_ *ProcessEvent) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *ProcessEvent) SetInt(v int32) { panic("Unsupported operation") } +func (_ *ProcessEvent) SetLong(v int64) { panic("Unsupported operation") } +func (_ *ProcessEvent) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *ProcessEvent) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *ProcessEvent) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *ProcessEvent) SetString(v string) { panic("Unsupported operation") } +func (_ *ProcessEvent) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *ProcessEvent) Get(i int) types.Field { + switch i { + case 0: + r.ProcOID = NewOID() + + return r.ProcOID + case 1: + return &types.Long{Target: &r.Ts} + case 2: + return &types.Long{Target: &r.Tid} + case 3: + return &types.Int{Target: &r.OpFlags} + case 4: + r.Args = make([]string, 0) + + return &ArrayStringWrapper{Target: &r.Args} + case 5: + return &types.Int{Target: &r.Ret} + } + panic("Unknown field index") +} + +func (r *ProcessEvent) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *ProcessEvent) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *ProcessEvent) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *ProcessEvent) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *ProcessEvent) Finalize() {} + +func (_ *ProcessEvent) AvroCRC64Fingerprint() []byte { + return []byte(ProcessEventAvroCRC64Fingerprint) +} diff --git a/go/sfgo/process_flow.go b/go/sfgo/process_flow.go new file mode 100644 index 0000000..07cb630 --- /dev/null +++ b/go/sfgo/process_flow.go @@ -0,0 +1,160 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type ProcessFlow struct { + ProcOID *OID `json:"procOID"` + + Ts int64 `json:"ts"` + + NumThreadsCloned int64 `json:"numThreadsCloned"` + + OpFlags int32 `json:"opFlags"` + + EndTs int64 `json:"endTs"` + + NumThreadsExited int64 `json:"numThreadsExited"` + + NumCloneErrors int64 `json:"numCloneErrors"` +} + +const ProcessFlowAvroCRC64Fingerprint = "\xfd\u070f\xa3o\xe4:\xfe" + +func NewProcessFlow() *ProcessFlow { + return &ProcessFlow{} +} + +func DeserializeProcessFlow(r io.Reader) (*ProcessFlow, error) { + t := NewProcessFlow() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeProcessFlowFromSchema(r io.Reader, schema string) (*ProcessFlow, error) { + t := NewProcessFlow() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeProcessFlow(r *ProcessFlow, w io.Writer) error { + var err error + err = writeOID(r.ProcOID, w) + if err != nil { + return err + } + err = vm.WriteLong(r.Ts, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumThreadsCloned, w) + if err != nil { + return err + } + err = vm.WriteInt(r.OpFlags, w) + if err != nil { + return err + } + err = vm.WriteLong(r.EndTs, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumThreadsExited, w) + if err != nil { + return err + } + err = vm.WriteLong(r.NumCloneErrors, w) + if err != nil { + return err + } + return err +} + +func (r *ProcessFlow) Serialize(w io.Writer) error { + return writeProcessFlow(r, w) +} + +func (r *ProcessFlow) Schema() string { + return "{\"fields\":[{\"name\":\"procOID\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"numThreadsCloned\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"endTs\",\"type\":\"long\"},{\"name\":\"numThreadsExited\",\"type\":\"long\"},{\"name\":\"numCloneErrors\",\"type\":\"long\"}],\"name\":\"sysflow.flow.ProcessFlow\",\"type\":\"record\"}" +} + +func (r *ProcessFlow) SchemaName() string { + return "sysflow.flow.ProcessFlow" +} + +func (_ *ProcessFlow) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *ProcessFlow) SetInt(v int32) { panic("Unsupported operation") } +func (_ *ProcessFlow) SetLong(v int64) { panic("Unsupported operation") } +func (_ *ProcessFlow) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *ProcessFlow) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *ProcessFlow) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *ProcessFlow) SetString(v string) { panic("Unsupported operation") } +func (_ *ProcessFlow) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *ProcessFlow) Get(i int) types.Field { + switch i { + case 0: + r.ProcOID = NewOID() + + return r.ProcOID + case 1: + return &types.Long{Target: &r.Ts} + case 2: + return &types.Long{Target: &r.NumThreadsCloned} + case 3: + return &types.Int{Target: &r.OpFlags} + case 4: + return &types.Long{Target: &r.EndTs} + case 5: + return &types.Long{Target: &r.NumThreadsExited} + case 6: + return &types.Long{Target: &r.NumCloneErrors} + } + panic("Unknown field index") +} + +func (r *ProcessFlow) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *ProcessFlow) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *ProcessFlow) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *ProcessFlow) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *ProcessFlow) Finalize() {} + +func (_ *ProcessFlow) AvroCRC64Fingerprint() []byte { + return []byte(ProcessFlowAvroCRC64Fingerprint) +} diff --git a/go/sfgo/sf_header.go b/go/sfgo/sf_header.go new file mode 100644 index 0000000..63dcf8d --- /dev/null +++ b/go/sfgo/sf_header.go @@ -0,0 +1,132 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type SFHeader struct { + Version int64 `json:"version"` + + Exporter string `json:"exporter"` + + Ip string `json:"ip"` +} + +const SFHeaderAvroCRC64Fingerprint = "\xbfY\xaa\xd9;\x1b@%" + +func NewSFHeader() *SFHeader { + return &SFHeader{} +} + +func DeserializeSFHeader(r io.Reader) (*SFHeader, error) { + t := NewSFHeader() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeSFHeaderFromSchema(r io.Reader, schema string) (*SFHeader, error) { + t := NewSFHeader() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeSFHeader(r *SFHeader, w io.Writer) error { + var err error + err = vm.WriteLong(r.Version, w) + if err != nil { + return err + } + err = vm.WriteString(r.Exporter, w) + if err != nil { + return err + } + err = vm.WriteString(r.Ip, w) + if err != nil { + return err + } + return err +} + +func (r *SFHeader) Serialize(w io.Writer) error { + return writeSFHeader(r, w) +} + +func (r *SFHeader) Schema() string { + return "{\"fields\":[{\"default\":2,\"name\":\"version\",\"type\":\"long\"},{\"name\":\"exporter\",\"type\":\"string\"},{\"default\":\"NA\",\"name\":\"ip\",\"type\":\"string\"}],\"name\":\"sysflow.entity.SFHeader\",\"type\":\"record\"}" +} + +func (r *SFHeader) SchemaName() string { + return "sysflow.entity.SFHeader" +} + +func (_ *SFHeader) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *SFHeader) SetInt(v int32) { panic("Unsupported operation") } +func (_ *SFHeader) SetLong(v int64) { panic("Unsupported operation") } +func (_ *SFHeader) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *SFHeader) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *SFHeader) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *SFHeader) SetString(v string) { panic("Unsupported operation") } +func (_ *SFHeader) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *SFHeader) Get(i int) types.Field { + switch i { + case 0: + return &types.Long{Target: &r.Version} + case 1: + return &types.String{Target: &r.Exporter} + case 2: + return &types.String{Target: &r.Ip} + } + panic("Unknown field index") +} + +func (r *SFHeader) SetDefault(i int) { + switch i { + case 0: + r.Version = 2 + return + case 2: + r.Ip = "NA" + return + } + panic("Unknown field index") +} + +func (r *SFHeader) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *SFHeader) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *SFHeader) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *SFHeader) Finalize() {} + +func (_ *SFHeader) AvroCRC64Fingerprint() []byte { + return []byte(SFHeaderAvroCRC64Fingerprint) +} diff --git a/go/sfgo/sf_object_state.go b/go/sfgo/sf_object_state.go new file mode 100644 index 0000000..06eeda1 --- /dev/null +++ b/go/sfgo/sf_object_state.go @@ -0,0 +1,125 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" +) + +type SFObjectState int32 + +const ( + SFObjectStateCREATED SFObjectState = 0 + SFObjectStateMODIFIED SFObjectState = 1 + SFObjectStateREUP SFObjectState = 2 +) + +func (e SFObjectState) String() string { + switch e { + case SFObjectStateCREATED: + return "CREATED" + case SFObjectStateMODIFIED: + return "MODIFIED" + case SFObjectStateREUP: + return "REUP" + } + return "unknown" +} + +func writeSFObjectState(r SFObjectState, w io.Writer) error { + return vm.WriteInt(int32(r), w) +} + +func NewSFObjectStateValue(raw string) (r SFObjectState, err error) { + switch raw { + case "CREATED": + return SFObjectStateCREATED, nil + case "MODIFIED": + return SFObjectStateMODIFIED, nil + case "REUP": + return SFObjectStateREUP, nil + } + + return -1, fmt.Errorf("invalid value for SFObjectState: '%s'", raw) +} + +func (b *SFObjectState) MarshalJSON() ([]byte, error) { + return json.Marshal([]byte(b.String())) +} + +func (b *SFObjectState) UnmarshalJSON(data []byte) error { + var stringVal string + err := json.Unmarshal(data, &stringVal) + if err != nil { + return err + } + val, err := NewSFObjectStateValue(stringVal) + *b = val + return err +} + +type SFObjectStateWrapper struct { + Target *SFObjectState +} + +func (b *SFObjectStateWrapper) SetBoolean(v bool) { + panic("Unable to assign boolean to int field") +} + +func (b *SFObjectStateWrapper) SetInt(v int32) { + *(b.Target) = SFObjectState(v) +} + +func (b *SFObjectStateWrapper) SetLong(v int64) { + panic("Unable to assign long to int field") +} + +func (b *SFObjectStateWrapper) SetFloat(v float32) { + panic("Unable to assign float to int field") +} + +func (b *SFObjectStateWrapper) SetUnionElem(v int64) { + panic("Unable to assign union elem to int field") +} + +func (b *SFObjectStateWrapper) SetDouble(v float64) { + panic("Unable to assign double to int field") +} + +func (b *SFObjectStateWrapper) SetBytes(v []byte) { + panic("Unable to assign bytes to int field") +} + +func (b *SFObjectStateWrapper) SetString(v string) { + panic("Unable to assign string to int field") +} + +func (b *SFObjectStateWrapper) Get(i int) types.Field { + panic("Unable to get field from int field") +} + +func (b *SFObjectStateWrapper) SetDefault(i int) { + panic("Unable to set default on int field") +} + +func (b *SFObjectStateWrapper) AppendMap(key string) types.Field { + panic("Unable to append map key to from int field") +} + +func (b *SFObjectStateWrapper) AppendArray() types.Field { + panic("Unable to append array element to from int field") +} + +func (b *SFObjectStateWrapper) NullField(int) { + panic("Unable to null field in int field") +} + +func (b *SFObjectStateWrapper) Finalize() {} diff --git a/go/sfgo/sys_flow.go b/go/sfgo/sys_flow.go new file mode 100644 index 0000000..863c182 --- /dev/null +++ b/go/sfgo/sys_flow.go @@ -0,0 +1,112 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "github.com/actgardner/gogen-avro/v7/compiler" + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" + "io" +) + +type SysFlow struct { + Rec *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow `json:"rec"` +} + +const SysFlowAvroCRC64Fingerprint = "\xf9\x1f\x01\xe5\x1e+\x90\x8e" + +func NewSysFlow() *SysFlow { + return &SysFlow{} +} + +func DeserializeSysFlow(r io.Reader) (*SysFlow, error) { + t := NewSysFlow() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func DeserializeSysFlowFromSchema(r io.Reader, schema string) (*SysFlow, error) { + t := NewSysFlow() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return nil, err + } + + err = vm.Eval(r, deser, t) + if err != nil { + return nil, err + } + return t, err +} + +func writeSysFlow(r *SysFlow, w io.Writer) error { + var err error + err = writeUnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow(r.Rec, w) + if err != nil { + return err + } + return err +} + +func (r *SysFlow) Serialize(w io.Writer) error { + return writeSysFlow(r, w) +} + +func (r *SysFlow) Schema() string { + return "{\"fields\":[{\"name\":\"rec\",\"type\":[{\"fields\":[{\"default\":2,\"name\":\"version\",\"type\":\"long\"},{\"name\":\"exporter\",\"type\":\"string\"},{\"default\":\"NA\",\"name\":\"ip\",\"type\":\"string\"}],\"name\":\"SFHeader\",\"namespace\":\"sysflow.entity\",\"type\":\"record\"},{\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"image\",\"type\":\"string\"},{\"name\":\"imageid\",\"type\":\"string\"},{\"name\":\"type\",\"type\":{\"name\":\"ContainerType\",\"namespace\":\"sysflow.type\",\"symbols\":[\"CT_DOCKER\",\"CT_LXC\",\"CT_LIBVIRT_LXC\",\"CT_MESOS\",\"CT_RKT\",\"CT_CUSTOM\",\"CT_CRI\",\"CT_CONTAINERD\",\"CT_CRIO\",\"CT_BPM\"],\"type\":\"enum\"}},{\"name\":\"privileged\",\"type\":\"boolean\"}],\"name\":\"Container\",\"namespace\":\"sysflow.entity\",\"type\":\"record\"},{\"fields\":[{\"name\":\"state\",\"type\":{\"name\":\"SFObjectState\",\"namespace\":\"sysflow.type\",\"symbols\":[\"CREATED\",\"MODIFIED\",\"REUP\"],\"type\":\"enum\"}},{\"name\":\"oid\",\"type\":{\"fields\":[{\"name\":\"createTS\",\"type\":\"long\"},{\"name\":\"hpid\",\"type\":\"long\"}],\"name\":\"OID\",\"namespace\":\"sysflow.type\",\"type\":\"record\"}},{\"name\":\"poid\",\"type\":[\"null\",\"sysflow.type.OID\"]},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"exe\",\"type\":\"string\"},{\"name\":\"exeArgs\",\"type\":\"string\"},{\"name\":\"uid\",\"type\":\"int\"},{\"name\":\"userName\",\"type\":\"string\"},{\"name\":\"gid\",\"type\":\"int\"},{\"name\":\"groupName\",\"type\":\"string\"},{\"name\":\"tty\",\"type\":\"boolean\"},{\"name\":\"containerId\",\"type\":[\"null\",\"string\"]},{\"default\":false,\"name\":\"entry\",\"type\":\"boolean\"}],\"name\":\"Process\",\"namespace\":\"sysflow.entity\",\"type\":\"record\"},{\"fields\":[{\"name\":\"state\",\"type\":\"sysflow.type.SFObjectState\"},{\"name\":\"oid\",\"type\":{\"name\":\"FOID\",\"namespace\":\"sysflow.type\",\"size\":20,\"type\":\"fixed\"}},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"restype\",\"type\":\"int\"},{\"name\":\"path\",\"type\":\"string\"},{\"name\":\"containerId\",\"type\":[\"null\",\"string\"]}],\"name\":\"File\",\"namespace\":\"sysflow.entity\",\"type\":\"record\"},{\"fields\":[{\"name\":\"procOID\",\"type\":\"sysflow.type.OID\"},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"items\":\"string\",\"type\":\"array\"}},{\"name\":\"ret\",\"type\":\"int\"}],\"name\":\"ProcessEvent\",\"namespace\":\"sysflow.event\",\"type\":\"record\"},{\"fields\":[{\"name\":\"procOID\",\"type\":\"sysflow.type.OID\"},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"endTs\",\"type\":\"long\"},{\"name\":\"sip\",\"type\":\"int\"},{\"name\":\"sport\",\"type\":\"int\"},{\"name\":\"dip\",\"type\":\"int\"},{\"name\":\"dport\",\"type\":\"int\"},{\"name\":\"proto\",\"type\":\"int\"},{\"name\":\"fd\",\"type\":\"int\"},{\"name\":\"numRRecvOps\",\"type\":\"long\"},{\"name\":\"numWSendOps\",\"type\":\"long\"},{\"name\":\"numRRecvBytes\",\"type\":\"long\"},{\"name\":\"numWSendBytes\",\"type\":\"long\"}],\"name\":\"NetworkFlow\",\"namespace\":\"sysflow.flow\",\"type\":\"record\"},{\"fields\":[{\"name\":\"procOID\",\"type\":\"sysflow.type.OID\"},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"openFlags\",\"type\":\"int\"},{\"name\":\"endTs\",\"type\":\"long\"},{\"name\":\"fileOID\",\"type\":\"sysflow.type.FOID\"},{\"name\":\"fd\",\"type\":\"int\"},{\"name\":\"numRRecvOps\",\"type\":\"long\"},{\"name\":\"numWSendOps\",\"type\":\"long\"},{\"name\":\"numRRecvBytes\",\"type\":\"long\"},{\"name\":\"numWSendBytes\",\"type\":\"long\"}],\"name\":\"FileFlow\",\"namespace\":\"sysflow.flow\",\"type\":\"record\"},{\"fields\":[{\"name\":\"procOID\",\"type\":\"sysflow.type.OID\"},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"fileOID\",\"type\":\"sysflow.type.FOID\"},{\"name\":\"ret\",\"type\":\"int\"},{\"name\":\"newFileOID\",\"type\":[\"null\",\"sysflow.type.FOID\"]}],\"name\":\"FileEvent\",\"namespace\":\"sysflow.event\",\"type\":\"record\"},{\"fields\":[{\"name\":\"procOID\",\"type\":\"sysflow.type.OID\"},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"tid\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"sip\",\"type\":\"int\"},{\"name\":\"sport\",\"type\":\"int\"},{\"name\":\"dip\",\"type\":\"int\"},{\"name\":\"dport\",\"type\":\"int\"},{\"name\":\"proto\",\"type\":\"int\"},{\"name\":\"ret\",\"type\":\"int\"}],\"name\":\"NetworkEvent\",\"namespace\":\"sysflow.event\",\"type\":\"record\"},{\"fields\":[{\"name\":\"procOID\",\"type\":\"sysflow.type.OID\"},{\"name\":\"ts\",\"type\":\"long\"},{\"name\":\"numThreadsCloned\",\"type\":\"long\"},{\"name\":\"opFlags\",\"type\":\"int\"},{\"name\":\"endTs\",\"type\":\"long\"},{\"name\":\"numThreadsExited\",\"type\":\"long\"},{\"name\":\"numCloneErrors\",\"type\":\"long\"}],\"name\":\"ProcessFlow\",\"namespace\":\"sysflow.flow\",\"type\":\"record\"}]}],\"name\":\"sysflow.SysFlow\",\"type\":\"record\"}" +} + +func (r *SysFlow) SchemaName() string { + return "sysflow.SysFlow" +} + +func (_ *SysFlow) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *SysFlow) SetInt(v int32) { panic("Unsupported operation") } +func (_ *SysFlow) SetLong(v int64) { panic("Unsupported operation") } +func (_ *SysFlow) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *SysFlow) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *SysFlow) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *SysFlow) SetString(v string) { panic("Unsupported operation") } +func (_ *SysFlow) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *SysFlow) Get(i int) types.Field { + switch i { + case 0: + r.Rec = NewUnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow() + + return r.Rec + } + panic("Unknown field index") +} + +func (r *SysFlow) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *SysFlow) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ *SysFlow) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *SysFlow) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *SysFlow) Finalize() {} + +func (_ *SysFlow) AvroCRC64Fingerprint() []byte { + return []byte(SysFlowAvroCRC64Fingerprint) +} diff --git a/go/sfgo/union_null_foid.go b/go/sfgo/union_null_foid.go new file mode 100644 index 0000000..3b3f297 --- /dev/null +++ b/go/sfgo/union_null_foid.go @@ -0,0 +1,96 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" +) + +type UnionNullFOIDTypeEnum int + +const ( + UnionNullFOIDTypeEnumFOID UnionNullFOIDTypeEnum = 1 +) + +type UnionNullFOID struct { + Null *types.NullVal + FOID FOID + UnionType UnionNullFOIDTypeEnum +} + +func writeUnionNullFOID(r *UnionNullFOID, w io.Writer) error { + + if r == nil { + err := vm.WriteLong(0, w) + return err + } + + err := vm.WriteLong(int64(r.UnionType), w) + if err != nil { + return err + } + switch r.UnionType { + case UnionNullFOIDTypeEnumFOID: + return writeFOID(r.FOID, w) + } + return fmt.Errorf("invalid value for *UnionNullFOID") +} + +func NewUnionNullFOID() *UnionNullFOID { + return &UnionNullFOID{} +} + +func (_ *UnionNullFOID) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *UnionNullFOID) SetInt(v int32) { panic("Unsupported operation") } +func (_ *UnionNullFOID) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *UnionNullFOID) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *UnionNullFOID) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *UnionNullFOID) SetString(v string) { panic("Unsupported operation") } +func (r *UnionNullFOID) SetLong(v int64) { + r.UnionType = (UnionNullFOIDTypeEnum)(v) +} +func (r *UnionNullFOID) Get(i int) types.Field { + switch i { + case 0: + return r.Null + case 1: + return &FOIDWrapper{Target: (&r.FOID)} + } + panic("Unknown field index") +} +func (_ *UnionNullFOID) NullField(i int) { panic("Unsupported operation") } +func (_ *UnionNullFOID) SetDefault(i int) { panic("Unsupported operation") } +func (_ *UnionNullFOID) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *UnionNullFOID) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *UnionNullFOID) Finalize() {} + +func (r *UnionNullFOID) MarshalJSON() ([]byte, error) { + if r == nil { + return []byte("null"), nil + } + switch r.UnionType { + case UnionNullFOIDTypeEnumFOID: + return json.Marshal(map[string]interface{}{"FOID": r.FOID}) + } + return nil, fmt.Errorf("invalid value for *UnionNullFOID") +} + +func (r *UnionNullFOID) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + if value, ok := fields["FOID"]; ok { + r.UnionType = 1 + return json.Unmarshal([]byte(value), &r.FOID) + } + return fmt.Errorf("invalid value for *UnionNullFOID") +} diff --git a/go/sfgo/union_null_oid.go b/go/sfgo/union_null_oid.go new file mode 100644 index 0000000..f14f06c --- /dev/null +++ b/go/sfgo/union_null_oid.go @@ -0,0 +1,97 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" +) + +type UnionNullOIDTypeEnum int + +const ( + UnionNullOIDTypeEnumOID UnionNullOIDTypeEnum = 1 +) + +type UnionNullOID struct { + Null *types.NullVal + OID *OID + UnionType UnionNullOIDTypeEnum +} + +func writeUnionNullOID(r *UnionNullOID, w io.Writer) error { + + if r == nil { + err := vm.WriteLong(0, w) + return err + } + + err := vm.WriteLong(int64(r.UnionType), w) + if err != nil { + return err + } + switch r.UnionType { + case UnionNullOIDTypeEnumOID: + return writeOID(r.OID, w) + } + return fmt.Errorf("invalid value for *UnionNullOID") +} + +func NewUnionNullOID() *UnionNullOID { + return &UnionNullOID{} +} + +func (_ *UnionNullOID) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *UnionNullOID) SetInt(v int32) { panic("Unsupported operation") } +func (_ *UnionNullOID) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *UnionNullOID) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *UnionNullOID) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *UnionNullOID) SetString(v string) { panic("Unsupported operation") } +func (r *UnionNullOID) SetLong(v int64) { + r.UnionType = (UnionNullOIDTypeEnum)(v) +} +func (r *UnionNullOID) Get(i int) types.Field { + switch i { + case 0: + return r.Null + case 1: + r.OID = NewOID() + return r.OID + } + panic("Unknown field index") +} +func (_ *UnionNullOID) NullField(i int) { panic("Unsupported operation") } +func (_ *UnionNullOID) SetDefault(i int) { panic("Unsupported operation") } +func (_ *UnionNullOID) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *UnionNullOID) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *UnionNullOID) Finalize() {} + +func (r *UnionNullOID) MarshalJSON() ([]byte, error) { + if r == nil { + return []byte("null"), nil + } + switch r.UnionType { + case UnionNullOIDTypeEnumOID: + return json.Marshal(map[string]interface{}{"OID": r.OID}) + } + return nil, fmt.Errorf("invalid value for *UnionNullOID") +} + +func (r *UnionNullOID) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + if value, ok := fields["OID"]; ok { + r.UnionType = 1 + return json.Unmarshal([]byte(value), &r.OID) + } + return fmt.Errorf("invalid value for *UnionNullOID") +} diff --git a/go/sfgo/union_null_string.go b/go/sfgo/union_null_string.go new file mode 100644 index 0000000..09e1b7f --- /dev/null +++ b/go/sfgo/union_null_string.go @@ -0,0 +1,96 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" +) + +type UnionNullStringTypeEnum int + +const ( + UnionNullStringTypeEnumString UnionNullStringTypeEnum = 1 +) + +type UnionNullString struct { + Null *types.NullVal + String string + UnionType UnionNullStringTypeEnum +} + +func writeUnionNullString(r *UnionNullString, w io.Writer) error { + + if r == nil { + err := vm.WriteLong(0, w) + return err + } + + err := vm.WriteLong(int64(r.UnionType), w) + if err != nil { + return err + } + switch r.UnionType { + case UnionNullStringTypeEnumString: + return vm.WriteString(r.String, w) + } + return fmt.Errorf("invalid value for *UnionNullString") +} + +func NewUnionNullString() *UnionNullString { + return &UnionNullString{} +} + +func (_ *UnionNullString) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ *UnionNullString) SetInt(v int32) { panic("Unsupported operation") } +func (_ *UnionNullString) SetFloat(v float32) { panic("Unsupported operation") } +func (_ *UnionNullString) SetDouble(v float64) { panic("Unsupported operation") } +func (_ *UnionNullString) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ *UnionNullString) SetString(v string) { panic("Unsupported operation") } +func (r *UnionNullString) SetLong(v int64) { + r.UnionType = (UnionNullStringTypeEnum)(v) +} +func (r *UnionNullString) Get(i int) types.Field { + switch i { + case 0: + return r.Null + case 1: + return &types.String{Target: (&r.String)} + } + panic("Unknown field index") +} +func (_ *UnionNullString) NullField(i int) { panic("Unsupported operation") } +func (_ *UnionNullString) SetDefault(i int) { panic("Unsupported operation") } +func (_ *UnionNullString) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ *UnionNullString) AppendArray() types.Field { panic("Unsupported operation") } +func (_ *UnionNullString) Finalize() {} + +func (r *UnionNullString) MarshalJSON() ([]byte, error) { + if r == nil { + return []byte("null"), nil + } + switch r.UnionType { + case UnionNullStringTypeEnumString: + return json.Marshal(map[string]interface{}{"string": r.String}) + } + return nil, fmt.Errorf("invalid value for *UnionNullString") +} + +func (r *UnionNullString) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + if value, ok := fields["string"]; ok { + r.UnionType = 1 + return json.Unmarshal([]byte(value), &r.String) + } + return fmt.Errorf("invalid value for *UnionNullString") +} diff --git a/go/sfgo/union_sf_header_container_process_file_process_event_network_flow_file_flow_file_event_network_event_process_flow.go b/go/sfgo/union_sf_header_container_process_file_process_event_network_flow_file_flow_file_event_network_event_process_flow.go new file mode 100644 index 0000000..897e758 --- /dev/null +++ b/go/sfgo/union_sf_header_container_process_file_process_event_network_flow_file_flow_file_event_network_event_process_flow.go @@ -0,0 +1,236 @@ +// Code generated by github.com/actgardner/gogen-avro/v7. DO NOT EDIT. +/* + * SOURCE: + * SysFlow.avsc + */ +package sfgo + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v7/vm" + "github.com/actgardner/gogen-avro/v7/vm/types" +) + +type UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum int + +const ( + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumSFHeader UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 0 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumContainer UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 1 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcess UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 2 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFile UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 3 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessEvent UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 4 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkFlow UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 5 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileFlow UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 6 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileEvent UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 7 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkEvent UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 8 + + UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessFlow UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum = 9 +) + +type UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow struct { + SFHeader *SFHeader + Container *Container + Process *Process + File *File + ProcessEvent *ProcessEvent + NetworkFlow *NetworkFlow + FileFlow *FileFlow + FileEvent *FileEvent + NetworkEvent *NetworkEvent + ProcessFlow *ProcessFlow + UnionType UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum +} + +func writeUnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow(r *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow, w io.Writer) error { + + err := vm.WriteLong(int64(r.UnionType), w) + if err != nil { + return err + } + switch r.UnionType { + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumSFHeader: + return writeSFHeader(r.SFHeader, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumContainer: + return writeContainer(r.Container, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcess: + return writeProcess(r.Process, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFile: + return writeFile(r.File, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessEvent: + return writeProcessEvent(r.ProcessEvent, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkFlow: + return writeNetworkFlow(r.NetworkFlow, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileFlow: + return writeFileFlow(r.FileFlow, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileEvent: + return writeFileEvent(r.FileEvent, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkEvent: + return writeNetworkEvent(r.NetworkEvent, w) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessFlow: + return writeProcessFlow(r.ProcessFlow, w) + } + return fmt.Errorf("invalid value for *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow") +} + +func NewUnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow() *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow { + return &UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow{} +} + +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetBoolean(v bool) { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetInt(v int32) { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetFloat(v float32) { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetDouble(v float64) { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetBytes(v []byte) { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetString(v string) { + panic("Unsupported operation") +} +func (r *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetLong(v int64) { + r.UnionType = (UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnum)(v) +} +func (r *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) Get(i int) types.Field { + switch i { + case 0: + r.SFHeader = NewSFHeader() + return r.SFHeader + case 1: + r.Container = NewContainer() + return r.Container + case 2: + r.Process = NewProcess() + return r.Process + case 3: + r.File = NewFile() + return r.File + case 4: + r.ProcessEvent = NewProcessEvent() + return r.ProcessEvent + case 5: + r.NetworkFlow = NewNetworkFlow() + return r.NetworkFlow + case 6: + r.FileFlow = NewFileFlow() + return r.FileFlow + case 7: + r.FileEvent = NewFileEvent() + return r.FileEvent + case 8: + r.NetworkEvent = NewNetworkEvent() + return r.NetworkEvent + case 9: + r.ProcessFlow = NewProcessFlow() + return r.ProcessFlow + } + panic("Unknown field index") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) NullField(i int) { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) SetDefault(i int) { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) AppendMap(key string) types.Field { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) AppendArray() types.Field { + panic("Unsupported operation") +} +func (_ *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) Finalize() { +} + +func (r *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) MarshalJSON() ([]byte, error) { + if r == nil { + return []byte("null"), nil + } + switch r.UnionType { + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumSFHeader: + return json.Marshal(map[string]interface{}{"SFHeader": r.SFHeader}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumContainer: + return json.Marshal(map[string]interface{}{"Container": r.Container}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcess: + return json.Marshal(map[string]interface{}{"Process": r.Process}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFile: + return json.Marshal(map[string]interface{}{"File": r.File}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessEvent: + return json.Marshal(map[string]interface{}{"ProcessEvent": r.ProcessEvent}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkFlow: + return json.Marshal(map[string]interface{}{"NetworkFlow": r.NetworkFlow}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileFlow: + return json.Marshal(map[string]interface{}{"FileFlow": r.FileFlow}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumFileEvent: + return json.Marshal(map[string]interface{}{"FileEvent": r.FileEvent}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumNetworkEvent: + return json.Marshal(map[string]interface{}{"NetworkEvent": r.NetworkEvent}) + case UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlowTypeEnumProcessFlow: + return json.Marshal(map[string]interface{}{"ProcessFlow": r.ProcessFlow}) + } + return nil, fmt.Errorf("invalid value for *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow") +} + +func (r *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + if value, ok := fields["SFHeader"]; ok { + r.UnionType = 0 + return json.Unmarshal([]byte(value), &r.SFHeader) + } + if value, ok := fields["Container"]; ok { + r.UnionType = 1 + return json.Unmarshal([]byte(value), &r.Container) + } + if value, ok := fields["Process"]; ok { + r.UnionType = 2 + return json.Unmarshal([]byte(value), &r.Process) + } + if value, ok := fields["File"]; ok { + r.UnionType = 3 + return json.Unmarshal([]byte(value), &r.File) + } + if value, ok := fields["ProcessEvent"]; ok { + r.UnionType = 4 + return json.Unmarshal([]byte(value), &r.ProcessEvent) + } + if value, ok := fields["NetworkFlow"]; ok { + r.UnionType = 5 + return json.Unmarshal([]byte(value), &r.NetworkFlow) + } + if value, ok := fields["FileFlow"]; ok { + r.UnionType = 6 + return json.Unmarshal([]byte(value), &r.FileFlow) + } + if value, ok := fields["FileEvent"]; ok { + r.UnionType = 7 + return json.Unmarshal([]byte(value), &r.FileEvent) + } + if value, ok := fields["NetworkEvent"]; ok { + r.UnionType = 8 + return json.Unmarshal([]byte(value), &r.NetworkEvent) + } + if value, ok := fields["ProcessFlow"]; ok { + r.UnionType = 9 + return json.Unmarshal([]byte(value), &r.ProcessFlow) + } + return fmt.Errorf("invalid value for *UnionSFHeaderContainerProcessFileProcessEventNetworkFlowFileFlowFileEventNetworkEventProcessFlow") +} diff --git a/go/utils/utils.go b/go/utils/utils.go new file mode 100644 index 0000000..01787a1 --- /dev/null +++ b/go/utils/utils.go @@ -0,0 +1,372 @@ +package utils + +import ( + "bytes" + "strconv" + "strings" + "time" + + "github.com/sysflow-telemetry/sf-apis/go/sfgo" +) + +const ( + nanoToSecs = 1000000000 + timeFormat = "2006-01-02T15:04:05-0700" +) + +// GetOpFlagsStr creates a string representation of opflags. +func GetOpFlagsStr(opFlags int32) string { + var b bytes.Buffer + b.WriteString(func() string { + if opFlags&sfgo.OP_MKDIR == sfgo.OP_MKDIR { + return "MKDIR" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_RMDIR == sfgo.OP_RMDIR { + return "RMDIR" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_LINK == sfgo.OP_LINK { + return "LINK" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_SYMLINK == sfgo.OP_SYMLINK { + return "SYMLINK" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_UNLINK == sfgo.OP_UNLINK { + return "UNLINK" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_RENAME == sfgo.OP_RENAME { + return "RENAME" + } + return "" + }()) + if b.Len() > 0 { + return b.String() + } + b.WriteString(func() string { + if opFlags&sfgo.OP_CLONE == sfgo.OP_CLONE { + return "CLONE" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_EXEC == sfgo.OP_EXEC { + return "EXEC" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_EXIT == sfgo.OP_EXIT { + return "EXIT" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_SETUID == sfgo.OP_SETUID { + return "SETUID" + } + return "" + }()) + if b.Len() > 0 { + return b.String() + } + b.WriteString(func() string { + if opFlags&sfgo.OP_OPEN == sfgo.OP_OPEN { + return "O" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_ACCEPT == sfgo.OP_ACCEPT { + return "A" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_CONNECT == sfgo.OP_CONNECT { + return "C" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_WRITE_SEND == sfgo.OP_WRITE_SEND { + return "W" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_READ_RECV == sfgo.OP_READ_RECV { + return "R" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_SETNS == sfgo.OP_SETNS { + return "N" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_MMAP == sfgo.OP_MMAP { + return "M" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_SHUTDOWN == sfgo.OP_SHUTDOWN { + return "S" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_CLOSE == sfgo.OP_CLOSE { + return "C" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_TRUNCATE == sfgo.OP_TRUNCATE { + return "T" + } + return "" + }()) + b.WriteString(func() string { + if opFlags&sfgo.OP_DIGEST == sfgo.OP_DIGEST { + return "D" + } + return "" + }()) + return b.String() +} + +// GetOpFlags creates a list representation of opflag strings. +func GetOpFlags(opFlags int32, rtype string) []string { + var ops = make([]string, 0) + if opFlags&sfgo.OP_MKDIR == sfgo.OP_MKDIR { + ops = append(ops, "MKDIR") + } + if opFlags&sfgo.OP_RMDIR == sfgo.OP_RMDIR { + ops = append(ops, "RMDIR") + } + if opFlags&sfgo.OP_LINK == sfgo.OP_LINK { + ops = append(ops, "LINK") + } + if opFlags&sfgo.OP_SYMLINK == sfgo.OP_SYMLINK { + ops = append(ops, "SYMLINK") + } + if opFlags&sfgo.OP_UNLINK == sfgo.OP_UNLINK { + ops = append(ops, "UNLINK") + } + if opFlags&sfgo.OP_RENAME == sfgo.OP_RENAME { + ops = append(ops, "RENAME") + } + if opFlags&sfgo.OP_CLONE == sfgo.OP_CLONE { + ops = append(ops, "CLONE") + } + if opFlags&sfgo.OP_EXEC == sfgo.OP_EXEC { + ops = append(ops, "EXEC") + } + if opFlags&sfgo.OP_EXIT == sfgo.OP_EXIT { + ops = append(ops, "EXIT") + } + if opFlags&sfgo.OP_SETUID == sfgo.OP_SETUID { + ops = append(ops, "SETUID") + } + if opFlags&sfgo.OP_OPEN == sfgo.OP_OPEN { + ops = append(ops, "OPEN") + } + if opFlags&sfgo.OP_ACCEPT == sfgo.OP_ACCEPT { + ops = append(ops, "ACCEPT") + } + if opFlags&sfgo.OP_CONNECT == sfgo.OP_CONNECT { + ops = append(ops, "CONNECT") + } + if opFlags&sfgo.OP_WRITE_SEND == sfgo.OP_WRITE_SEND { + if rtype == "NF" { + ops = append(ops, "SEND") + } else { + ops = append(ops, "WRITE") + } + } + if opFlags&sfgo.OP_READ_RECV == sfgo.OP_READ_RECV { + if rtype == "NF" { + ops = append(ops, "RECV") + } else { + ops = append(ops, "READ") + } + } + if opFlags&sfgo.OP_SETNS == sfgo.OP_SETNS { + ops = append(ops, "SETNS") + } + if opFlags&sfgo.OP_MMAP == sfgo.OP_MMAP { + ops = append(ops, "MMAP") + } + if opFlags&sfgo.OP_SHUTDOWN == sfgo.OP_SHUTDOWN { + ops = append(ops, "SHUTDOWN") + } + if opFlags&sfgo.OP_TRUNCATE == sfgo.OP_TRUNCATE { + ops = append(ops, "TRUNCATE") + } + if opFlags&sfgo.OP_DIGEST == sfgo.OP_DIGEST { + ops = append(ops, "DIGEST") + } + return ops +} + +// GetOpenFlags converts a sysflow open modes flag bitmap into a slice representation. +func GetOpenFlags(flag int64) []string { + var flags = make([]string, 0) + if flag&sfgo.O_NONE == sfgo.O_NONE { + flags = append(flags, "NONE") + } + if flag&sfgo.O_RDONLY == sfgo.O_RDONLY { + flags = append(flags, "RDONLY") + } + if flag&sfgo.O_WRONLY == sfgo.O_WRONLY { + flags = append(flags, "WRONLY") + } + if flag&sfgo.O_RDWR == sfgo.O_RDWR { + flags = append(flags, "RDWR") + } + if flag&sfgo.O_CREAT == sfgo.O_CREAT { + flags = append(flags, "CREAT") + } + if flag&sfgo.O_EXCL == sfgo.O_EXCL { + flags = append(flags, "EXCL") + } + if flag&sfgo.O_TRUNC == sfgo.O_TRUNC { + flags = append(flags, "TRUNC") + } + if flag&sfgo.O_APPEND == sfgo.O_APPEND { + flags = append(flags, "APPEND") + } + if flag&sfgo.O_NONBLOCK == sfgo.O_NONBLOCK { + flags = append(flags, "NONBLOCK") + } + if flag&sfgo.O_DSYNC == sfgo.O_DSYNC { + flags = append(flags, "DSYNC") + } + if flag&sfgo.O_DIRECT == sfgo.O_DIRECT { + flags = append(flags, "DIRECT") + } + if flag&sfgo.O_LARGEFILE == sfgo.O_LARGEFILE { + flags = append(flags, "LARGEFILE") + } + if flag&sfgo.O_DIRECTORY == sfgo.O_DIRECTORY { + flags = append(flags, "DIRECTORY") + } + if flag&sfgo.O_CLOEXEC == sfgo.O_CLOEXEC { + flags = append(flags, "CLOEXEC") + } + if flag&sfgo.O_SYNC == sfgo.O_SYNC { + flags = append(flags, "SYNC") + } + return flags +} + +// IsOpenRead checks if file flags is open for read. +func IsOpenRead(flag int64) bool { + return flag&sfgo.O_RDWR == sfgo.O_RDWR || flag&sfgo.O_RDONLY == sfgo.O_RDONLY +} + +// IsOpenWrite checks if file flags is open for write. +func IsOpenWrite(flag int64) bool { + return flag&sfgo.O_RDWR == sfgo.O_RDWR || flag&sfgo.O_WRONLY == sfgo.O_WRONLY +} + +// GetContType returns string representing container type. +func GetContType(t int64) string { + return strings.ReplaceAll(sfgo.ContainerType(t).String(), "CT_", "") +} + +// GetProto returns the string representation of a L4 network protocol provided in IANA format. +func GetProto(iana int64) string { + switch iana { + case 6: + return "tcp" + case 17: + return "udp" + case 1: + return "icmp" + case 254: + return "raw" + default: + break + } + return sfgo.Zeros.String +} + +// GetFileType returns the string representation of a ASCII file type. +func GetFileType(t int64) string { + return string(t) +} + +// GetSockFamily returns the sock family of a socket descriptor. +func GetSockFamily(t int64) string { + switch GetFileType(t) { + case "4": + case "6": + return "ip" + case "u": + return "unix" + default: + break + } + return sfgo.Zeros.String +} + +// GetTimeStrLocal creates a formatted timestamp from a unix timestamp. +func GetTimeStrLocal(unix int64) string { + tm := time.Unix(unix/nanoToSecs, 0).Local() + return tm.Format(timeFormat) +} + +// GetTimeStrUTC creates a UTC timestamp from a unix timestamp. +func GetTimeStrUTC(unix int64) string { + tm := time.Unix(unix/nanoToSecs, 0).UTC() + return tm.Format(timeFormat) +} + +// GetTimeUTC creates a UTC timestamp from a unix timestamp. +func GetTimeUTC(unix int64) time.Time { + return time.Unix(unix/nanoToSecs, 0).UTC() +} + +// GetIPStr creates a string representation of an IP address. +func GetIPStr(ip int32) string { + var b bytes.Buffer + b.WriteString(strconv.Itoa(int(ip >> 0 & 0xFF))) + b.WriteString(".") + b.WriteString(strconv.Itoa(int(ip >> 8 & 0xFF))) + b.WriteString(".") + b.WriteString(strconv.Itoa(int(ip >> 16 & 0xFF))) + b.WriteString(".") + b.WriteString(strconv.Itoa(int(ip >> 24 & 0xFF))) + return b.String() +} + +// GetNetworkFlowStr creates a string representation out of a newtoork flow. +func GetNetworkFlowStr(nf *sfgo.NetworkFlow) string { + var b bytes.Buffer + b.WriteString(GetIPStr(nf.Sip)) + b.WriteString(":") + b.WriteString(strconv.Itoa(int(nf.Sport))) + b.WriteString("-") + b.WriteString(GetIPStr(nf.Dip)) + b.WriteString(":") + b.WriteString(strconv.Itoa(int(nf.Dport))) + return b.String() +} diff --git a/py3/classes/sysflow/__init__.py b/py3/classes/sysflow/__init__.py index cef7b85..140d627 100644 --- a/py3/classes/sysflow/__init__.py +++ b/py3/classes/sysflow/__init__.py @@ -1,34 +1,35 @@ -from .schema_classes import SchemaClasses, SCHEMA as my_schema, get_schema_type -from avro.io import DatumReader +#from .schema_classes import SchemaClasses, SCHEMA as my_schema, get_schema_type +#from avro.io import DatumReader -class SpecificDatumReader(DatumReader): - SCHEMA_TYPES = { - "sysflow.SysFlow": SchemaClasses.sysflow.SysFlowClass, - "sysflow.entity.Container": SchemaClasses.sysflow.entity.ContainerClass, - "sysflow.entity.File": SchemaClasses.sysflow.entity.FileClass, - "sysflow.entity.Process": SchemaClasses.sysflow.entity.ProcessClass, - "sysflow.entity.SFHeader": SchemaClasses.sysflow.entity.SFHeaderClass, - "sysflow.event.FileEvent": SchemaClasses.sysflow.event.FileEventClass, - "sysflow.event.NetworkEvent": SchemaClasses.sysflow.event.NetworkEventClass, - "sysflow.event.ProcessEvent": SchemaClasses.sysflow.event.ProcessEventClass, - "sysflow.flow.FileFlow": SchemaClasses.sysflow.flow.FileFlowClass, - "sysflow.flow.NetworkFlow": SchemaClasses.sysflow.flow.NetworkFlowClass, - "sysflow.type.ContainerType": SchemaClasses.sysflow.type.ContainerTypeClass, - "sysflow.type.OID": SchemaClasses.sysflow.type.OIDClass, - "sysflow.type.SFObjectState": SchemaClasses.sysflow.type.SFObjectStateClass, - } - def __init__(self, readers_schema=None, **kwargs): - writers_schema = kwargs.pop("writers_schema", readers_schema) - writers_schema = kwargs.pop("writer_schema", writers_schema) - super(SpecificDatumReader, self).__init__(writers_schema, readers_schema, **kwargs) - def read_record(self, writers_schema, readers_schema, decoder): +#class SpecificDatumReader(DatumReader): +# SCHEMA_TYPES = { +# "sysflow.SysFlow": SchemaClasses.sysflow.SysFlowClass, +# "sysflow.entity.Container": SchemaClasses.sysflow.entity.ContainerClass, +# "sysflow.entity.File": SchemaClasses.sysflow.entity.FileClass, +# "sysflow.entity.Process": SchemaClasses.sysflow.entity.ProcessClass, +# "sysflow.entity.SFHeader": SchemaClasses.sysflow.entity.SFHeaderClass, +# "sysflow.event.FileEvent": SchemaClasses.sysflow.event.FileEventClass, +# "sysflow.event.NetworkEvent": SchemaClasses.sysflow.event.NetworkEventClass, +# "sysflow.event.ProcessEvent": SchemaClasses.sysflow.event.ProcessEventClass, +# "sysflow.flow.FileFlow": SchemaClasses.sysflow.flow.FileFlowClass, +# "sysflow.flow.NetworkFlow": SchemaClasses.sysflow.flow.NetworkFlowClass, +# "sysflow.flow.ProcessFlow": SchemaClasses.sysflow.flow.ProcessFlowClass, +# "sysflow.type.ContainerType": SchemaClasses.sysflow.type.ContainerTypeClass, +# "sysflow.type.OID": SchemaClasses.sysflow.type.OIDClass, +# "sysflow.type.SFObjectState": SchemaClasses.sysflow.type.SFObjectStateClass, +# } +# def __init__(self, readers_schema=None, **kwargs): +# writers_schema = kwargs.pop("writers_schema", readers_schema) +# writers_schema = kwargs.pop("writer_schema", writers_schema) +# super(SpecificDatumReader, self).__init__(writers_schema, readers_schema, **kwargs) +# def read_record(self, writers_schema, readers_schema, decoder): - result = super(SpecificDatumReader, self).read_record(writers_schema, readers_schema, decoder) +# result = super(SpecificDatumReader, self).read_record(writers_schema, readers_schema, decoder) - if readers_schema.fullname in SpecificDatumReader.SCHEMA_TYPES: - result = SpecificDatumReader.SCHEMA_TYPES[readers_schema.fullname](result) +# if readers_schema.fullname in SpecificDatumReader.SCHEMA_TYPES: +# result = SpecificDatumReader.SCHEMA_TYPES[readers_schema.fullname](result) - return result \ No newline at end of file +# return result diff --git a/py3/classes/sysflow/formatter.py b/py3/classes/sysflow/formatter.py index 03f413f..1915b18 100644 --- a/py3/classes/sysflow/formatter.py +++ b/py3/classes/sysflow/formatter.py @@ -36,12 +36,12 @@ .. moduleauthor:: Frederico Araujo, Teryl Taylor """ -_version = '0.1-rc3' +_version = '0.1.0-rc4' _default_fields = ['ts_uts', 'type', 'proc.exe', 'proc.args', 'pproc.pid', 'proc.pid', 'proc.tid', 'opflags', 'res', 'flow.rbytes', 'flow.wbytes', 'container.id'] _fields = { # '': (, , , ) - 'idx': ('Rec #', 6, 'Record number', False), + 'idx': ('Rec #', 6, 'Record number', False), 'type': ('T', 2, 'Record type', False), 'state': ('State', 12, 'Entity state', False), 'opflags': ('Op Flags', 14, 'Operation flags', False), @@ -64,20 +64,20 @@ 'proc.name': ('Proc. Name', 20, 'Process name (query only)', True), 'proc.cmdline': ('Cmd Line', 20, 'Process command line (query only)', True), 'proc.tty': ('TTY', 5, 'Process TTY status', False), - 'proc.createts': ('Proc. Creation Time', 21, 'Process creation timestamp', False), - 'proc.duration': ('Proc. Duration', 8, 'Process duration/time from creation (query only)', True), + 'proc.entry': ('Entry', 5, 'Process container entrypoint', False), + 'proc.createts': ('Proc. Creation Time', 21, 'Process creation timestamp', False), 'pproc.pid': ('PPID', 8, 'Parent process ID', False), 'pproc.gid': ('PGID', 5, 'Parent process group ID', False), 'pproc.uid': ('PUID', 5, 'Parent process user ID', False), 'pproc.group': ('PGroup', 8, 'Parent process group name', False), - 'pproc.tty': ('PTTY', 5, 'Parent process TTY status', False), + 'pproc.tty': ('PTTY', 5, 'Parent process TTY status', False), + 'pproc.entry': ('PEntry', 5, 'Parent process container entry', False), 'pproc.user': ('PUser', 8, 'Parent process user name', False), 'pproc.exe': ('PCmd', 20, 'Parent process command/filename', False), 'pproc.args': ('PArgs', 20, 'Parent process command arguments', False), 'pproc.name': ('PProc. Name', 20, 'Parent process name (query only)', True), 'pproc.cmdline': ('PProc. Cmd Line', 20, 'Parent process command line (query only)', True), - 'pproc.createts': ('PProc. Creation Time', 21, 'Parent process creation timestamp', False), - 'pproc.duration': ('PProc. Duration', 8, 'Process duration/time from creation (query only)', True), + 'pproc.createts': ('PProc. Creation Time', 21, 'Parent process creation timestamp', False), 'file.fd': ('FD', 5, 'File descriptor number', False), 'file.path': ('Path', 30, 'File path', False), 'file.newpath': ('New Path', 30, 'New file path', False), @@ -101,10 +101,14 @@ 'flow.wops': ('NoOpsWrite', 8, 'Flow bytes written/sent', False), 'container.id': ('Cont ID', 12, 'Container ID', False), 'container.name': ('Cont Name', 12, 'Container name', False), - 'container.imageid': ('Image ID', 12, 'Container image ID', False), + 'container.imageid': ('Image ID', 12, 'Container image ID', False), 'container.image': ('Image Name', 12, 'Container image name', False), 'container.type': ('Cont Type', 8, 'Container type', False), - 'container.privileged': ('Privileged', 5, 'Container privilege status', False) + 'container.privileged': ('Privileged', 5, 'Container privilege status', False), + 'node.id': ('Node ID', 12, 'Node identifier', False), + 'node.ip': ('Node IP', 16, 'Node IP address', False), + 'schema': ('SF Schema', 8, 'SysFlow schema version', False), + 'version': ('API version', 8, 'SysFlow JSON schema version', False), } class SFFormatter(object): @@ -324,15 +328,15 @@ def _get_terminal_size(self, fallback=(80, 24)): def _flatten(self, objtype, header, cont, pproc, proc, files, evt, flow, fields): _flat_map = OrderedDict() evflow = evt or flow - _flat_map['v'] = _version + _flat_map['version'] = _version _flat_map['type'] = OBJECT_MAP.get(objtype,'?') _flat_map['state'] = proc.state if proc else files[0].state if files and files[0] else '' _flat_map['opflags'] = utils.getOpFlagsStr(evflow.opFlags) if evflow else '' _flat_map['opflags_bitmap'] = evflow.opFlags if evflow else '' _flat_map['ret'] = int(evflow.ret) if evt else None - _flat_map['ts'] = utils.getTimeStr(evflow.ts) if evflow else '' + _flat_map['ts'] = utils.getTimeStrIso8601(evflow.ts) if evflow else '' _flat_map['ts_uts'] = int(evflow.ts) if evflow else None - _flat_map['endts'] = utils.getTimeStr(evflow.endTs) if flow else '' + _flat_map['endts'] = utils.getTimeStrIso8601(evflow.endTs) if flow else '' _flat_map['endts_uts'] = int(evflow.endTs) if flow else None _flat_map['proc.pid'] = int(proc.oid.hpid) if proc else None _flat_map['proc.tid'] = int(evflow.tid) if evflow else None @@ -343,12 +347,14 @@ def _flatten(self, objtype, header, cont, pproc, proc, files, evt, flow, fields) _flat_map['proc.exe'] = proc.exe if proc else '' _flat_map['proc.args'] = proc.exeArgs if proc else '' _flat_map['proc.tty'] = proc.tty if proc else '' + _flat_map['proc.entry'] = proc.entry if proc and hasattr(proc, 'entry') else '' _flat_map['proc.createts'] = int(proc.oid.createTS) if proc else None _flat_map['pproc.pid'] = int(pproc.oid.hpid) if pproc else None _flat_map['pproc.gid'] = int(pproc.gid) if pproc else None _flat_map['pproc.uid'] = int(pproc.uid) if pproc else None _flat_map['pproc.group'] = pproc.groupName if pproc else '' _flat_map['pproc.tty'] = pproc.tty if pproc else '' + _flat_map['pproc.entry'] = pproc.entry if pproc and hasattr(pproc, 'entry') else '' _flat_map['pproc.user'] = pproc.userName if pproc else '' _flat_map['pproc.exe'] = pproc.exe if pproc else '' _flat_map['pproc.args'] = pproc.exeArgs if pproc else '' @@ -382,6 +388,9 @@ def _flatten(self, objtype, header, cont, pproc, proc, files, evt, flow, fields) _flat_map['container.image'] = cont.image if cont else '' _flat_map['container.type'] = cont.type if cont else '' _flat_map['container.privileged'] = cont.privileged if cont else '' + _flat_map['node.id'] = header.exporter if header else '' + _flat_map['node.ip'] = header.ip if header and hasattr(header, 'ip') else '' + _flat_map['schema'] = header.version if header else '' if fields: od = OrderedDict() diff --git a/py3/classes/sysflow/objtypes.py b/py3/classes/sysflow/objtypes.py index 0a824dc..5abdaa5 100644 --- a/py3/classes/sysflow/objtypes.py +++ b/py3/classes/sysflow/objtypes.py @@ -48,6 +48,7 @@ class ObjectTypes(Enum): NET_FLOW = 5 FILE_FLOW = 6 FILE_EVT = 7 + PROC_FLOW = 8 OBJECT_MAP = { ObjectTypes.HEADER: "H", @@ -57,5 +58,18 @@ class ObjectTypes(Enum): ObjectTypes.PROC_EVT : "PE", ObjectTypes.NET_FLOW : "NF", ObjectTypes.FILE_FLOW : "FF", - ObjectTypes.FILE_EVT : "FE" + ObjectTypes.FILE_EVT : "FE", + ObjectTypes.PROC_FLOW : "PF" } + +OBJ_NAME_MAP = { + "sysflow.entity.SFHeader": ObjectTypes.HEADER, + "sysflow.entity.Container": ObjectTypes.CONT, + "sysflow.entity.Process": ObjectTypes.PROC, + "sysflow.entity.File": ObjectTypes.FILE, + "sysflow.event.ProcessEvent": ObjectTypes.PROC_EVT, + "sysflow.event.FileEvent": ObjectTypes.FILE_EVT, + "sysflow.flow.NetworkFlow": ObjectTypes.NET_FLOW, + "sysflow.flow.FileFlow": ObjectTypes.FILE_FLOW, + "sysflow.flow.ProcessFlow": ObjectTypes.PROC_FLOW +} diff --git a/py3/classes/sysflow/openflags.py b/py3/classes/sysflow/openflags.py index 1a984cd..82d43f8 100644 --- a/py3/classes/sysflow/openflags.py +++ b/py3/classes/sysflow/openflags.py @@ -20,52 +20,38 @@ # """ .. module:: sysflow.openflags - :synopsis: This module lists all open operations as defined in fcntl.h. + :synopsis: This module lists all open operations as defined by sysdig's conversions. .. moduleauthor:: Frederico Araujo, Teryl Taylor """ """ - O_RDONLY = (0) - O_WRONLY = (1) - O_RDWR = (2) - O_ACCMODE = (3) - O_CREAT = (1 << 6) - O_EXCL = (1 << 7) - O_NOCTTY = (1 << 8) - O_TRUNC = (1 << 9) - O_APPEND = (1 << 10) - O_NONBLOCK = (1 << 11) - O_NDELAY = O_NONBLOCK - O_DSYNC = (1 << 12) - O_FASYNC = (1 << 13) - O_DIRECT = (1 << 14) - O_LARGEFILE = (1 << 15) - O_DIRECTORY = (1 << 16) - O_NOFOLLOW = (1 << 17) - O_NOATIME = (1 << 18) - O_CLOEXEC = (1 << 19) - O_SYNC = (1 << 20 | O_DSYNC) - O_PATH = (1 << 21) - O_TMPFILE = (1 << 22) + O_NONE = (0) + O_RDONLY = (1 << 0) # Open for reading only + O_WRONLY = (1 << 1) # Open for writing only + O_RDWR = (PPM_O_RDONLY | PPM_O_WRONLY) # Open for reading and writing + O_CREAT = (1 << 2) # Create a new file if it doesn't exist. + O_APPEND = (1 << 3) # If set, the file offset shall be set to the end of the file prior to each write. + O_DSYNC = (1 << 4) + O_EXCL = (1 << 5) + O_NONBLOCK = (1 << 6) + O_SYNC = (1 << 7) + O_TRUNC = (1 << 8) + O_DIRECT = (1 << 9) + O_DIRECTORY = (1 << 10) + O_LARGEFILE = (1 << 11) + O_CLOEXEC = (1 << 12) """ -O_RDONLY = (0) -O_WRONLY = (1) -O_RDWR = (2) -O_ACCMODE = (3) -O_CREAT = (1 << 6) -O_EXCL = (1 << 7) -O_NOCTTY = (1 << 8) -O_TRUNC = (1 << 9) -O_APPEND = (1 << 10) -O_NONBLOCK = (1 << 11) -O_NDELAY = O_NONBLOCK -O_DSYNC = (1 << 12) -O_FASYNC = (1 << 13) -O_DIRECT = (1 << 14) -O_LARGEFILE = (1 << 15) -O_DIRECTORY = (1 << 16) -O_NOFOLLOW = (1 << 17) -O_NOATIME = (1 << 18) -O_CLOEXEC = (1 << 19) -O_SYNC = (1 << 20 | O_DSYNC) -O_PATH = (1 << 21) -O_TMPFILE = (1 << 22) \ No newline at end of file +O_NONE = (0) +O_RDONLY = (1 << 0) # Open for reading only +O_WRONLY = (1 << 1) # Open for writing only +O_RDWR = (O_RDONLY | O_WRONLY) # Open for reading and writing +O_CREAT = (1 << 2) # Create a new file if it doesn't exist. +O_APPEND = (1 << 3) # If set, the file offset shall be set to the end of the file prior to each write. +O_DSYNC = (1 << 4) +O_EXCL = (1 << 5) +O_NONBLOCK = (1 << 6) +O_SYNC = (1 << 7) +O_TRUNC = (1 << 8) +O_DIRECT = (1 << 9) +O_DIRECTORY = (1 << 10) +O_LARGEFILE = (1 << 11) +O_CLOEXEC = (1 << 12) diff --git a/py3/classes/sysflow/reader.py b/py3/classes/sysflow/reader.py index f2bf172..13571a7 100644 --- a/py3/classes/sysflow/reader.py +++ b/py3/classes/sysflow/reader.py @@ -18,15 +18,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import avro.io -import io -import sysflow -from sysflow.objtypes import ObjectTypes -from sysflow import SpecificDatumReader as SysFlowReader -from avro import datafile, io -from sysflow.schema_classes import SCHEMA as SysFlowSchema -from avro import datafile, io +from sysflow.objtypes import ObjectTypes, OBJ_NAME_MAP from uuid import UUID +from fastavro import reader +from types import SimpleNamespace """ .. module:: sysflow.reader @@ -34,6 +29,47 @@ .. moduleauthor:: Frederico Araujo, Teryl Taylor """ + +class NestedNamespace(SimpleNamespace): + + @staticmethod + def mapEntry(entry): + if isinstance(entry, dict): + return NestedNamespace(**entry) + return entry + + def __init__(self, **kwargs): + super().__init__(**kwargs) + for key, val in kwargs.items(): + if isinstance(val, dict): + setattr(self, key, NestedNamespace(**val)) + elif isinstance(val, list): + setattr(self, key, list(map(self.mapEntry, val))) + elif isinstance(val, tuple): + if len(val) == 2: + obj = val[1] + if isinstance(obj, dict): + setattr(self, key, NestedNamespace(**obj)) + else: + setattr(self, key, obj) + else: + setattr(self, key, tuple(map(self.mapEntry, val))) + +def modifySchema(schema): + union = schema['fields'][0]['type'] + for obj in union: + removeLogicalTypes(obj) + +def removeLogicalTypes(obj): + objFields = obj['fields'] + for t in objFields: + if isinstance(t['type'],dict): + if 'logicalType' in t['type']: + t['type'].pop('logicalType') + elif 'fields' in t['type']: + removeLogicalTypes(t['type']) + + class SFReader(object): """ **SFReader** @@ -44,13 +80,12 @@ class SFReader(object): Example Usage:: reader = SFReader("./sysflowfile.sf") - for sf in reader: - rec = sf.rec - if isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.entity.SFHeaderClass): + for name, sf in reader: + if name == "sysflow.entity.SFHeader": //do something with the header object - elif isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.entity.ContainerClass): + elif name == "sysflow.entity.Container": //do something with the container object - elif isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.entity.ProcessClass): + elif name == "sysflow.entity.Process": //do something with the Process object .... @@ -60,21 +95,23 @@ class SFReader(object): def __init__(self, filename): self.filename = filename self.fh = open(filename, "rb") - self.reader = datafile.DataFileReader(self.fh, SysFlowReader(readers_schema=SysFlowSchema)) + self.rdr = reader(self.fh, return_record_name=True) + modifySchema(self.rdr.writer_schema) def __iter__(self): return self def next(self): - return next(self.reader) - + record = next(self.rdr) + name, obj = record['rec'] + o = NestedNamespace(**obj) + return OBJ_NAME_MAP[name], o + def __next__(self): return self.next() def close(self): - self.reader.close() - - + self.rdr.close() class FlattenedSFReader(SFReader): """ @@ -85,27 +122,29 @@ class FlattenedSFReader(SFReader): This class supports the python iterator design pattern. Example Usage:: - reader = FlattenedSFReader("./sysflowfile.sf", False) - for objtype, header, cont, proc, files, evt, flow in reader: - exe = proc.exe + ' ' + proc.exeArgs if proc is not None else '' - pid = proc.oid.hpid if proc is not None else '' - evflow = evt or flow - tid = evflow.tid if evflow is not None else '' - opFlags = utils.getOpFlagsStr(evflow.opFlags) if evflow is not None else '' - sTime = utils.getTimeStr(evflow.ts) if evflow is not None else '' - eTime = utils.getTimeStr(evflow.endTs) if flow is not None else '' - ret = evflow.ret if evt is not None else '' - res1 = '' - if objtype == ObjectTypes.FILE_FLOW or objtype == ObjectTypes.FILE_EVT: - res1 = files[0].path - elif objtype == ObjectTypes.NET_FLOW: - res1 = utils.getNetFlowStr(flow) - numBReads = evflow.numRRecvBytes if flow is not None else '' - numBWrites = evflow.numWSendBytes if flow is not None else '' - - res2 = files[1].path if files is not None and files[1] is not None else '' - cont = cont.id if cont is not None else '' - print("|{0:30}|{1:9}|{2:26}|{3:26}|{4:30}|{5:8}|{6:8}|".format(exe, opFlags, sTime, eTime, res1, numBReads, numBWrites)) + reader = FlattenedSFReader(trace) + head = 20 # max number of records to print + for i, (objtype, header, cont, pproc, proc, files, evt, flow) in enumerate(reader): + exe = proc.exe + pid = proc.oid.hpid if proc else '' + evflow = evt or flow + tid = evflow.tid if evflow else '' + opFlags = utils.getOpFlagsStr(evflow.opFlags) if evflow else '' + sTime = utils.getTimeStr(evflow.ts) if evflow else '' + eTime = utils.getTimeStr(evflow.endTs) if flow else '' + ret = evflow.ret if evt else '' + res1 = '' + if objtype == ObjectTypes.FILE_FLOW or objtype == ObjectTypes.FILE_EVT: + res1 = files[0].path + elif objtype == ObjectTypes.NET_FLOW: + res1 = utils.getNetFlowStr(flow) + numBReads = evflow.numRRecvBytes if flow else '' + numBWrites = evflow.numWSendBytes if flow else '' + res2 = files[1].path if files and files[1] else '' + cont = cont.id if cont else '' + print("|{0:30}|{1:9}|{2:26}|{3:26}|{4:30}|{5:8}|{6:8}|".format(exe, opFlags, sTime, eTime, res1, numBReads, numBWrites)) + if i == head: + break :param filename: the name of the sysflow file to be read. :type filename: str @@ -122,6 +161,8 @@ class FlattenedSFReader(SFReader): **cont** (:class:`sysflow.entity.Container`) The container associated with the flow/evt, or None if no container. + **pproc** (:class:`sysflow.entity.Process`) The parent process associated with the flow/evt. + **proc** (:class:`sysflow.entity.Process`) The process associated with the flow/evt. **files** (tuple of :class:`sysflow.entity.File`) Any files associated with the flow/evt. @@ -162,36 +203,35 @@ def getProcessKey(self, oid): def __next__(self): while True: - sf = super().next() - rec = sf.rec - if isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.entity.SFHeaderClass): + objtype, rec = super().next() + if objtype == ObjectTypes.HEADER: self.header = rec if self.retEntities: return (ObjectTypes.HEADER, rec, None, None, None, None, None, None) - elif isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.entity.ContainerClass): + elif objtype == ObjectTypes.CONT: key = rec.id self.conts[key] = rec if self.retEntities: return (ObjectTypes.CONT, self.header, rec, None, None, None, None, None) - elif isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.entity.ProcessClass): + elif objtype == ObjectTypes.PROC: key = self.getProcessKey(rec.oid) self.processes[key] = rec if self.retEntities: container = None if rec.containerId is not None: if not rec.containerId in self.conts: - print("ERROR: Cannot find container object for record. This should not happen.") + print("ERROR: Cannot find container object for record. This should not happen.") else: container = self.conts[rec.containerId] return (ObjectTypes.PROC, self.header, container, None, rec, None, None, None) - elif isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.entity.FileClass): + elif objtype == ObjectTypes.FILE: key = rec.oid self.files[key] = rec if self.retEntities: container = None if rec.containerId is not None: if not rec.containerId in self.conts: - print("ERROR: Cannot find container object for record. This should not happen.") + print("ERROR: Cannot find container object for record. This should not happen.") else: container = self.conts[rec.containerId] return (ObjectTypes.FILE, self.header, container, None, None, (rec, None), None, None) @@ -204,7 +244,6 @@ def __next__(self): file2 = None evt = None flow = None - objType = ObjectTypes.NET_FLOW if not procOID in self.processes: print("ERROR: Cannot find process object for record. This should not happen.") else: @@ -216,10 +255,9 @@ def __next__(self): print("ERROR: Cannot find container object for record. This should not happen.") else: container = self.conts[proc.containerId] - if isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.event.FileEventClass): + if objtype == ObjectTypes.FILE_EVT: fileOID = rec.fileOID evt = rec - objType = ObjectTypes.FILE_EVT if not fileOID in self.files: print("ERROR: Cannot find file object for record. This should not happen.") else: @@ -231,18 +269,16 @@ def __next__(self): else: file2 = self.files[fileOID2] - elif isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.flow.FileFlowClass): - objType = ObjectTypes.FILE_FLOW + elif objtype == ObjectTypes.FILE_FLOW: fileOID = rec.fileOID flow = rec if not fileOID in self.files: print("ERROR: Cannot find file object for record. This should not happen.") else: file1 = self.files[fileOID] - elif isinstance(rec, sysflow.schema_classes.SchemaClasses.sysflow.event.ProcessEventClass): - objType = ObjectTypes.PROC_EVT + elif objtype == ObjectTypes.PROC_EVT: evt = rec else: flow = rec files = (file1, file2) if file1 is not None or file2 is not None else None - return (objType, self.header, container, pproc, proc, files, evt, flow) + return (objtype, self.header, container, pproc, proc, files, evt, flow) diff --git a/py3/classes/sysflow/schema.avsc b/py3/classes/sysflow/schema.avsc index 7a18bf6..6f014ac 100644 --- a/py3/classes/sysflow/schema.avsc +++ b/py3/classes/sysflow/schema.avsc @@ -1 +1 @@ -{ "type" : "record", "name" : "SysFlow", "namespace" : "sysflow", "fields" : [ { "name" : "rec", "type" : [ { "type" : "record", "name" : "SFHeader", "namespace" : "sysflow.entity", "fields" : [ { "name" : "version", "type" : "long" }, { "name" : "exporter", "type" : "string" } ] }, { "type" : "record", "name" : "Container", "namespace" : "sysflow.entity", "fields" : [ { "name" : "id", "type" : "string" }, { "name" : "name", "type" : "string" }, { "name" : "image", "type" : "string" }, { "name" : "imageid", "type" : "string" }, { "name" : "type", "type" : { "type" : "enum", "name" : "ContainerType", "namespace" : "sysflow.type", "symbols" : [ "CT_DOCKER", "CT_LXC", "CT_LIBVIRT_LXC", "CT_MESOS", "CT_RKT", "CT_CUSTOM", "CT_CRI", "CT_CONTAINERD", "CT_CRIO", "CT_BPM" ] } }, { "name" : "privileged", "type" : "boolean" } ] }, { "type" : "record", "name" : "Process", "namespace" : "sysflow.entity", "fields" : [ { "name" : "state", "type" : { "type" : "enum", "name" : "SFObjectState", "namespace" : "sysflow.type", "symbols" : [ "CREATED", "MODIFIED", "REUP" ] } }, { "name" : "oid", "type" : { "type" : "record", "name" : "OID", "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "hpid", "type" : "long" } ] } }, { "name" : "poid", "type" : [ "null", "sysflow.type.OID" ] }, { "name" : "ts", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "exe", "type" : "string" }, { "name" : "exeArgs", "type" : "string" }, { "name" : "uid", "type" : "int" }, { "name" : "userName", "type" : "string" }, { "name" : "gid", "type" : "int" }, { "name" : "groupName", "type" : "string" }, { "name" : "tty", "type" : "boolean" }, { "name" : "containerId", "type" : [ "null", "string" ] } ] }, { "type" : "record", "name" : "File", "namespace" : "sysflow.entity", "fields" : [ { "name" : "state", "type" : "sysflow.type.SFObjectState" }, { "name" : "oid", "type" : { "type" : "fixed", "name" : "FOID", "namespace" : "sysflow.type", "size" : 20 } }, { "name" : "ts", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "restype", "type" : "int" }, { "name" : "path", "type" : "string" }, { "name" : "containerId", "type" : [ "null", "string" ] } ] }, { "type" : "record", "name" : "ProcessEvent", "namespace" : "sysflow.event", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "args", "type" : { "type" : "array", "items" : "string" } }, { "name" : "ret", "type" : "int" } ] }, { "type" : "record", "name" : "NetworkFlow", "namespace" : "sysflow.flow", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "endTs", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "sip", "type" : "int" }, { "name" : "sport", "type" : "int" }, { "name" : "dip", "type" : "int" }, { "name" : "dport", "type" : "int" }, { "name" : "proto", "type" : "int" }, { "name" : "fd", "type" : "int" }, { "name" : "numRRecvOps", "type" : "long" }, { "name" : "numWSendOps", "type" : "long" }, { "name" : "numRRecvBytes", "type" : "long" }, { "name" : "numWSendBytes", "type" : "long" } ] }, { "type" : "record", "name" : "FileFlow", "namespace" : "sysflow.flow", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "openFlags", "type" : "int" }, { "name" : "endTs", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "fileOID", "type" : "sysflow.type.FOID" }, { "name" : "fd", "type" : "int" }, { "name" : "numRRecvOps", "type" : "long" }, { "name" : "numWSendOps", "type" : "long" }, { "name" : "numRRecvBytes", "type" : "long" }, { "name" : "numWSendBytes", "type" : "long" } ] }, { "type" : "record", "name" : "FileEvent", "namespace" : "sysflow.event", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "fileOID", "type" : "sysflow.type.FOID" }, { "name" : "ret", "type" : "int" }, { "name" : "newFileOID", "type" : [ "null", "sysflow.type.FOID" ] } ] }, { "type" : "record", "name" : "NetworkEvent", "namespace" : "sysflow.event", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : { "type" : "long", "logicalType" : "timestamp-millis" } }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "sip", "type" : "int" }, { "name" : "sport", "type" : "int" }, { "name" : "dip", "type" : "int" }, { "name" : "dport", "type" : "int" }, { "name" : "proto", "type" : "int" }, { "name" : "ret", "type" : "int" } ] } ] } ]} \ No newline at end of file +{ "type" : "record", "name" : "SysFlow", "namespace" : "sysflow", "fields" : [ { "name" : "rec", "type" : [ { "type" : "record", "name" : "SFHeader", "namespace" : "sysflow.entity", "fields" : [ { "name" : "version", "type" : "long", "default" : 2 }, { "name" : "exporter", "type" : "string" }, { "name" : "ip", "type" : "string", "default" : "NA" } ] }, { "type" : "record", "name" : "Container", "namespace" : "sysflow.entity", "fields" : [ { "name" : "id", "type" : "string" }, { "name" : "name", "type" : "string" }, { "name" : "image", "type" : "string" }, { "name" : "imageid", "type" : "string" }, { "name" : "type", "type" : { "type" : "enum", "name" : "ContainerType", "namespace" : "sysflow.type", "symbols" : [ "CT_DOCKER", "CT_LXC", "CT_LIBVIRT_LXC", "CT_MESOS", "CT_RKT", "CT_CUSTOM", "CT_CRI", "CT_CONTAINERD", "CT_CRIO", "CT_BPM" ] } }, { "name" : "privileged", "type" : "boolean" } ] }, { "type" : "record", "name" : "Process", "namespace" : "sysflow.entity", "fields" : [ { "name" : "state", "type" : { "type" : "enum", "name" : "SFObjectState", "namespace" : "sysflow.type", "symbols" : [ "CREATED", "MODIFIED", "REUP" ] } }, { "name" : "oid", "type" : { "type" : "record", "name" : "OID", "namespace" : "sysflow.type", "fields" : [ { "name" : "createTS", "type" : "long" }, { "name" : "hpid", "type" : "long" } ] } }, { "name" : "poid", "type" : [ "null", "sysflow.type.OID" ] }, { "name" : "ts", "type" : "long" }, { "name" : "exe", "type" : "string" }, { "name" : "exeArgs", "type" : "string" }, { "name" : "uid", "type" : "int" }, { "name" : "userName", "type" : "string" }, { "name" : "gid", "type" : "int" }, { "name" : "groupName", "type" : "string" }, { "name" : "tty", "type" : "boolean" }, { "name" : "containerId", "type" : [ "null", "string" ] }, { "name" : "entry", "type" : "boolean", "default" : false } ] }, { "type" : "record", "name" : "File", "namespace" : "sysflow.entity", "fields" : [ { "name" : "state", "type" : "sysflow.type.SFObjectState" }, { "name" : "oid", "type" : { "type" : "fixed", "name" : "FOID", "namespace" : "sysflow.type", "size" : 20 } }, { "name" : "ts", "type" : "long" }, { "name" : "restype", "type" : "int" }, { "name" : "path", "type" : "string" }, { "name" : "containerId", "type" : [ "null", "string" ] } ] }, { "type" : "record", "name" : "ProcessEvent", "namespace" : "sysflow.event", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : "long" }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "args", "type" : { "type" : "array", "items" : "string" } }, { "name" : "ret", "type" : "int" } ] }, { "type" : "record", "name" : "NetworkFlow", "namespace" : "sysflow.flow", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : "long" }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "endTs", "type" : "long" }, { "name" : "sip", "type" : "int" }, { "name" : "sport", "type" : "int" }, { "name" : "dip", "type" : "int" }, { "name" : "dport", "type" : "int" }, { "name" : "proto", "type" : "int" }, { "name" : "fd", "type" : "int" }, { "name" : "numRRecvOps", "type" : "long" }, { "name" : "numWSendOps", "type" : "long" }, { "name" : "numRRecvBytes", "type" : "long" }, { "name" : "numWSendBytes", "type" : "long" } ] }, { "type" : "record", "name" : "FileFlow", "namespace" : "sysflow.flow", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : "long" }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "openFlags", "type" : "int" }, { "name" : "endTs", "type" : "long" }, { "name" : "fileOID", "type" : "sysflow.type.FOID" }, { "name" : "fd", "type" : "int" }, { "name" : "numRRecvOps", "type" : "long" }, { "name" : "numWSendOps", "type" : "long" }, { "name" : "numRRecvBytes", "type" : "long" }, { "name" : "numWSendBytes", "type" : "long" } ] }, { "type" : "record", "name" : "FileEvent", "namespace" : "sysflow.event", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : "long" }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "fileOID", "type" : "sysflow.type.FOID" }, { "name" : "ret", "type" : "int" }, { "name" : "newFileOID", "type" : [ "null", "sysflow.type.FOID" ] } ] }, { "type" : "record", "name" : "NetworkEvent", "namespace" : "sysflow.event", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : "long" }, { "name" : "tid", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "sip", "type" : "int" }, { "name" : "sport", "type" : "int" }, { "name" : "dip", "type" : "int" }, { "name" : "dport", "type" : "int" }, { "name" : "proto", "type" : "int" }, { "name" : "ret", "type" : "int" } ] }, { "type" : "record", "name" : "ProcessFlow", "namespace" : "sysflow.flow", "fields" : [ { "name" : "procOID", "type" : "sysflow.type.OID" }, { "name" : "ts", "type" : "long" }, { "name" : "numThreadsCloned", "type" : "long" }, { "name" : "opFlags", "type" : "int" }, { "name" : "endTs", "type" : "long" }, { "name" : "numThreadsExited", "type" : "long" }, { "name" : "numCloneErrors", "type" : "long" } ] } ] } ]} \ No newline at end of file diff --git a/py3/classes/sysflow/schema_classes.py b/py3/classes/sysflow/schema_classes.py index 081fbae..fd0038a 100644 --- a/py3/classes/sysflow/schema_classes.py +++ b/py3/classes/sysflow/schema_classes.py @@ -54,14 +54,14 @@ def __init__(self, inner_dict=None): @property def rec(self): """ - :rtype: SchemaClasses.sysflow.entity.SFHeaderClass | SchemaClasses.sysflow.entity.ContainerClass | SchemaClasses.sysflow.entity.ProcessClass | SchemaClasses.sysflow.entity.FileClass | SchemaClasses.sysflow.event.ProcessEventClass | SchemaClasses.sysflow.flow.NetworkFlowClass | SchemaClasses.sysflow.flow.FileFlowClass | SchemaClasses.sysflow.event.FileEventClass | SchemaClasses.sysflow.event.NetworkEventClass + :rtype: SchemaClasses.sysflow.entity.SFHeaderClass | SchemaClasses.sysflow.entity.ContainerClass | SchemaClasses.sysflow.entity.ProcessClass | SchemaClasses.sysflow.entity.FileClass | SchemaClasses.sysflow.event.ProcessEventClass | SchemaClasses.sysflow.flow.NetworkFlowClass | SchemaClasses.sysflow.flow.FileFlowClass | SchemaClasses.sysflow.event.FileEventClass | SchemaClasses.sysflow.event.NetworkEventClass | SchemaClasses.sysflow.flow.ProcessFlowClass """ return self._inner_dict.get('rec') @rec.setter def rec(self, value): #""" - #:param SchemaClasses.sysflow.entity.SFHeaderClass | SchemaClasses.sysflow.entity.ContainerClass | SchemaClasses.sysflow.entity.ProcessClass | SchemaClasses.sysflow.entity.FileClass | SchemaClasses.sysflow.event.ProcessEventClass | SchemaClasses.sysflow.flow.NetworkFlowClass | SchemaClasses.sysflow.flow.FileFlowClass | SchemaClasses.sysflow.event.FileEventClass | SchemaClasses.sysflow.event.NetworkEventClass value: + #:param SchemaClasses.sysflow.entity.SFHeaderClass | SchemaClasses.sysflow.entity.ContainerClass | SchemaClasses.sysflow.entity.ProcessClass | SchemaClasses.sysflow.entity.FileClass | SchemaClasses.sysflow.event.ProcessEventClass | SchemaClasses.sysflow.flow.NetworkFlowClass | SchemaClasses.sysflow.flow.FileFlowClass | SchemaClasses.sysflow.event.FileEventClass | SchemaClasses.sysflow.event.NetworkEventClass | SchemaClasses.sysflow.flow.ProcessFlowClass value: #""" self._inner_dict['rec'] = value @@ -195,6 +195,7 @@ def __init__(self, inner_dict=None): if inner_dict is None: self.state = SchemaClasses.sysflow.type.SFObjectStateClass.CREATED self.oid = str() + self.ts = int() self.restype = int() self.path = str() self.containerId = None @@ -306,6 +307,7 @@ def __init__(self, inner_dict=None): self.state = SchemaClasses.sysflow.type.SFObjectStateClass.CREATED self.oid = SchemaClasses.sysflow.type.OIDClass() self.poid = None + self.ts = int() self.exe = str() self.exeArgs = str() self.uid = int() @@ -314,6 +316,7 @@ def __init__(self, inner_dict=None): self.groupName = str() self.tty = bool() self.containerId = None + self.entry = SchemaClasses.sysflow.entity.ProcessClass.RECORD_SCHEMA.fields[12].default @property @@ -496,6 +499,21 @@ def containerId(self, value): self._inner_dict['containerId'] = value + @property + def entry(self): + """ + :rtype: bool + """ + return self._inner_dict.get('entry') + + @entry.setter + def entry(self, value): + #""" + #:param bool value: + #""" + self._inner_dict['entry'] = value + + class SFHeaderClass(DictWrapper): """ @@ -509,8 +527,9 @@ class SFHeaderClass(DictWrapper): def __init__(self, inner_dict=None): super(SchemaClasses.sysflow.entity.SFHeaderClass, self).__init__(inner_dict) if inner_dict is None: - self.version = int() + self.version = SchemaClasses.sysflow.entity.SFHeaderClass.RECORD_SCHEMA.fields[0].default self.exporter = str() + self.ip = SchemaClasses.sysflow.entity.SFHeaderClass.RECORD_SCHEMA.fields[2].default @property @@ -543,6 +562,21 @@ def exporter(self, value): self._inner_dict['exporter'] = value + @property + def ip(self): + """ + :rtype: str + """ + return self._inner_dict.get('ip') + + @ip.setter + def ip(self, value): + #""" + #:param str value: + #""" + self._inner_dict['ip'] = value + + pass class event(object): @@ -560,6 +594,7 @@ def __init__(self, inner_dict=None): super(SchemaClasses.sysflow.event.FileEventClass, self).__init__(inner_dict) if inner_dict is None: self.procOID = SchemaClasses.sysflow.type.OIDClass() + self.ts = int() self.tid = int() self.opFlags = int() self.fileOID = str() @@ -686,6 +721,7 @@ def __init__(self, inner_dict=None): super(SchemaClasses.sysflow.event.NetworkEventClass, self).__init__(inner_dict) if inner_dict is None: self.procOID = SchemaClasses.sysflow.type.OIDClass() + self.ts = int() self.tid = int() self.opFlags = int() self.sip = int() @@ -860,6 +896,7 @@ def __init__(self, inner_dict=None): super(SchemaClasses.sysflow.event.ProcessEventClass, self).__init__(inner_dict) if inner_dict is None: self.procOID = SchemaClasses.sysflow.type.OIDClass() + self.ts = int() self.tid = int() self.opFlags = int() self.args = list() @@ -973,9 +1010,11 @@ def __init__(self, inner_dict=None): super(SchemaClasses.sysflow.flow.FileFlowClass, self).__init__(inner_dict) if inner_dict is None: self.procOID = SchemaClasses.sysflow.type.OIDClass() + self.ts = int() self.tid = int() self.opFlags = int() self.openFlags = int() + self.endTs = int() self.fileOID = str() self.fd = int() self.numRRecvOps = int() @@ -1178,8 +1217,10 @@ def __init__(self, inner_dict=None): super(SchemaClasses.sysflow.flow.NetworkFlowClass, self).__init__(inner_dict) if inner_dict is None: self.procOID = SchemaClasses.sysflow.type.OIDClass() + self.ts = int() self.tid = int() self.opFlags = int() + self.endTs = int() self.sip = int() self.sport = int() self.dip = int() @@ -1417,6 +1458,133 @@ def numWSendBytes(self, value): self._inner_dict['numWSendBytes'] = value + class ProcessFlowClass(DictWrapper): + + """ + + """ + + + RECORD_SCHEMA = get_schema_type("sysflow.flow.ProcessFlow") + + + def __init__(self, inner_dict=None): + super(SchemaClasses.sysflow.flow.ProcessFlowClass, self).__init__(inner_dict) + if inner_dict is None: + self.procOID = SchemaClasses.sysflow.type.OIDClass() + self.ts = int() + self.numThreadsCloned = int() + self.opFlags = int() + self.endTs = int() + self.numThreadsExited = int() + self.numCloneErrors = int() + + + @property + def procOID(self): + """ + :rtype: SchemaClasses.sysflow.type.OIDClass + """ + return self._inner_dict.get('procOID') + + @procOID.setter + def procOID(self, value): + #""" + #:param SchemaClasses.sysflow.type.OIDClass value: + #""" + self._inner_dict['procOID'] = value + + + @property + def ts(self): + """ + :rtype: int + """ + return self._inner_dict.get('ts') + + @ts.setter + def ts(self, value): + #""" + #:param int value: + #""" + self._inner_dict['ts'] = value + + + @property + def numThreadsCloned(self): + """ + :rtype: int + """ + return self._inner_dict.get('numThreadsCloned') + + @numThreadsCloned.setter + def numThreadsCloned(self, value): + #""" + #:param int value: + #""" + self._inner_dict['numThreadsCloned'] = value + + + @property + def opFlags(self): + """ + :rtype: int + """ + return self._inner_dict.get('opFlags') + + @opFlags.setter + def opFlags(self, value): + #""" + #:param int value: + #""" + self._inner_dict['opFlags'] = value + + + @property + def endTs(self): + """ + :rtype: int + """ + return self._inner_dict.get('endTs') + + @endTs.setter + def endTs(self, value): + #""" + #:param int value: + #""" + self._inner_dict['endTs'] = value + + + @property + def numThreadsExited(self): + """ + :rtype: int + """ + return self._inner_dict.get('numThreadsExited') + + @numThreadsExited.setter + def numThreadsExited(self, value): + #""" + #:param int value: + #""" + self._inner_dict['numThreadsExited'] = value + + + @property + def numCloneErrors(self): + """ + :rtype: int + """ + return self._inner_dict.get('numCloneErrors') + + @numCloneErrors.setter + def numCloneErrors(self, value): + #""" + #:param int value: + #""" + self._inner_dict['numCloneErrors'] = value + + pass class type(object): @@ -1450,6 +1618,7 @@ class OIDClass(DictWrapper): def __init__(self, inner_dict=None): super(SchemaClasses.sysflow.type.OIDClass, self).__init__(inner_dict) if inner_dict is None: + self.createTS = int() self.hpid = int() @@ -1506,6 +1675,7 @@ class SFObjectStateClass(object): 'sysflow.event.ProcessEvent': SchemaClasses.sysflow.event.ProcessEventClass, 'sysflow.flow.FileFlow': SchemaClasses.sysflow.flow.FileFlowClass, 'sysflow.flow.NetworkFlow': SchemaClasses.sysflow.flow.NetworkFlowClass, + 'sysflow.flow.ProcessFlow': SchemaClasses.sysflow.flow.ProcessFlowClass, 'sysflow.type.ContainerType': SchemaClasses.sysflow.type.ContainerTypeClass, 'sysflow.type.OID': SchemaClasses.sysflow.type.OIDClass, 'sysflow.type.SFObjectState': SchemaClasses.sysflow.type.SFObjectStateClass, diff --git a/py3/classes/sysflow/sfql.py b/py3/classes/sysflow/sfql.py index be9779f..506eec0 100644 --- a/py3/classes/sysflow/sfql.py +++ b/py3/classes/sysflow/sfql.py @@ -21,6 +21,7 @@ import os, time from functools import reduce, partial from typing import Callable, Generic, TypeVar +from frozendict import frozendict from antlr4 import CommonTokenStream, FileStream, InputStream, ParseTreeWalker from sysflow.grammar.sfqlLexer import sfqlLexer from sysflow.grammar.sfqlListener import sfqlListener @@ -265,9 +266,19 @@ def _getPathBasename(path: str): def _getObjType(t: T, attr: str = None): return OBJECT_MAP.get(t[0],'?') + + @staticmethod + def _getHeaderAttr(t: T, attr: str): + hd = t[1] + if not hd: + return None + return SfqlMapper._rgetattr(hd, attr) + @staticmethod def _getContAttr(t: T, attr: str): cont = t[2] + if not cont: + return None return SfqlMapper._rgetattr(cont, attr) @staticmethod @@ -283,11 +294,11 @@ def _getEvtFlowAttr(t: T, attr: str): @staticmethod def _getProcAttr(t: T, attr: str): proc = t[4] - if attr == 'duration': - return int(time.time()) - int(proc.oid.createTs) + if not proc: + return None elif attr == 'cmdline': return proc.exe + ' ' + proc.exeArgs - elif attr == 'apid': + elif attr == 'apid': apid = SfqlMapper._getProcAncestry(proc.oid, 'oid.hpid', [proc.oid.hpid]) return ','.join([str(i) for i in apid]) elif attr == 'aname': @@ -298,14 +309,15 @@ def _getProcAttr(t: T, attr: str): @staticmethod def _getProcAncestry(oid, attr: str, anc: list): - pproc = SfqlMapper._ptree[oid] if oid in SfqlMapper._ptree else None + _oid = frozendict(vars(oid)) + pproc = SfqlMapper._ptree[_oid] if _oid in SfqlMapper._ptree else None return SfqlMapper._getProcAncestry(pproc.oid, attr, anc + [SfqlMapper._rgetattr(pproc, attr)]) if pproc else anc @staticmethod def _getPProcAttr(t: T, attr: str): proc = t[3] - if attr == 'duration': - return int(time.time()) - int(proc.oid.createTs) + if not proc: + return None elif attr == 'cmdline': return proc.exe + ' ' + proc.exeArgs else: @@ -330,7 +342,7 @@ def _getFileAttr(t: T, attr: str): @staticmethod def _getFileFlowAttr(t: T, attr: str): evflow = t[6] or t[7] - if t[0] != ObjectTypes.FILE_FLOW: + if t[0] != ObjectTypes.FILE_FLOW or not evflow: return None if attr == 'openFlags': return ','.join(utils.getOpenFlags(SfqlMapper._rgetattr(evflow, attr))) @@ -346,7 +358,7 @@ def _getFileFlowAttr(t: T, attr: str): @staticmethod def _getNetFlowAttr(t: T, attr: str): evflow = t[6] or t[7] - if t[0] != ObjectTypes.NET_FLOW: + if t[0] != ObjectTypes.NET_FLOW or not evflow: return None if attr == 'ip': return ','.join([SfqlMapper._rgetattr(evflow, 'sip'), SfqlMapper._rgetattr(evflow, 'dip')]) @@ -370,9 +382,9 @@ def _getNetFlowAttr(t: T, attr: str): 'proc.tid': partial(_getEvtFlowAttr.__func__, attr='tid'), 'proc.gid': partial(_getProcAttr.__func__, attr='gid'), 'proc.group': partial(_getProcAttr.__func__, attr='groupName'), - 'proc.createts': partial(_getProcAttr.__func__, attr='oid.createTS'), - 'proc.duration': partial(_getProcAttr.__func__, attr='duration'), + 'proc.createts': partial(_getProcAttr.__func__, attr='oid.createTS'), 'proc.tty': partial(_getProcAttr.__func__, attr='tty'), + 'proc.entry': partial(_getProcAttr.__func__, attr='entry'), 'proc.cmdline': partial(_getProcAttr.__func__, attr='cmdline'), 'proc.aname': partial(_getProcAttr.__func__, attr='aname'), 'proc.apid': partial(_getProcAttr.__func__, attr='apid'), @@ -385,8 +397,8 @@ def _getNetFlowAttr(t: T, attr: str): 'pproc.gid': partial(_getPProcAttr.__func__, attr='gid'), 'pproc.group': partial(_getPProcAttr.__func__, attr='groupName'), 'pproc.createts': partial(_getPProcAttr.__func__, attr='oid.createTS'), - 'pproc.duration': partial(_getPProcAttr.__func__, attr='duration'), 'pproc.tty': partial(_getPProcAttr.__func__, attr='tty'), + 'pproc.entry': partial(_getPProcAttr.__func__, attr='entry'), 'pproc.cmdline': partial(_getPProcAttr.__func__, attr='cmdline'), 'file.name': partial(_getFileAttr.__func__, attr='name'), 'file.path': partial(_getFileAttr.__func__, attr='path'), @@ -410,10 +422,13 @@ def _getNetFlowAttr(t: T, attr: str): 'flow.wops': partial(_getEvtFlowAttr.__func__, attr='numWSendOps'), 'container.id': partial(_getContAttr.__func__, attr='id'), 'container.name': partial(_getContAttr.__func__, attr='name'), - 'container.imageid': partial(_getContAttr.__func__, attr='imageid'), + 'container.imageid': partial(_getContAttr.__func__, attr='imageid'), 'container.image': partial(_getContAttr.__func__, attr='image'), 'container.type': partial(_getContAttr.__func__, attr='type'), - 'container.privileged': partial(_getContAttr.__func__, attr='privileged') + 'container.privileged': partial(_getContAttr.__func__, attr='privileged'), + 'node.id': partial(_getHeaderAttr.__func__, attr='exporter'), + 'node.ip': partial(_getHeaderAttr.__func__, attr='ip'), + 'schema': partial(_getHeaderAttr.__func__, attr='version') } def __init__(self): @@ -424,7 +439,8 @@ def hasAttr(self, attr: str): def getAttr(self, t: T, attr: str): if self.hasAttr(attr): - self._ptree[frozenset(t[4].oid)] = t[3] + if t[4]: + self._ptree[frozendict(vars(t[4].oid))] = t[3] return self._mapper[attr](t) else: return attr.strip('\"') diff --git a/py3/classes/sysflow/sysflow/flow/__init__.py b/py3/classes/sysflow/sysflow/flow/__init__.py index 9f2585b..1f96706 100644 --- a/py3/classes/sysflow/sysflow/flow/__init__.py +++ b/py3/classes/sysflow/sysflow/flow/__init__.py @@ -1,3 +1,4 @@ from ...schema_classes import SchemaClasses FileFlow = SchemaClasses.sysflow.flow.FileFlowClass NetworkFlow = SchemaClasses.sysflow.flow.NetworkFlowClass +ProcessFlow = SchemaClasses.sysflow.flow.ProcessFlowClass diff --git a/py3/classes/sysflow/utils.py b/py3/classes/sysflow/utils.py index f8a690d..01f322e 100644 --- a/py3/classes/sysflow/utils.py +++ b/py3/classes/sysflow/utils.py @@ -119,6 +119,7 @@ def getOpFlags(opFlags): if (opFlags & opflags.OP_RENAME): ops.add("RENAME") if (opFlags & opflags.OP_CLONE): ops.add("CLONE") if (opFlags & opflags.OP_EXEC): ops.add("EXEC") + if (opFlags & opflags.OP_EXIT): ops.add("EXIT") if (opFlags & opflags.OP_SETUID): ops.add("SETUID") if (opFlags & opflags.OP_OPEN): ops.add("OPEN") if (opFlags & opflags.OP_ACCEPT): ops.add("ACCEPT") @@ -159,28 +160,21 @@ def getOpenFlags(openFlags): :return: A set representation of the open modes bitmap. """ ops = set() + if (openFlags & openflags.O_NONE): ops.add("NONE") if (openFlags & openflags.O_RDONLY): ops.add("RDONLY") if (openFlags & openflags.O_WRONLY): ops.add("WRONLY") if (openFlags & openflags.O_RDWR): ops.add("RDWR") - if (openFlags & openflags.O_ACCMODE): ops.add("ACCMODE") if (openFlags & openflags.O_CREAT): ops.add("CREAT") if (openFlags & openflags.O_EXCL): ops.add("EXCL") - if (openFlags & openflags.O_NOCTTY): ops.add("NOCTTY") if (openFlags & openflags.O_TRUNC): ops.add("TRUNC") if (openFlags & openflags.O_APPEND): ops.add("APPEND") - if (openFlags & openflags.O_NONBLOCK): ops.add("NONBLOCK") - if (openFlags & openflags.O_NDELAY): ops.add("NDELAY") - if (openFlags & openflags.O_DSYNC): ops.add("DSYNC") - if (openFlags & openflags.O_FASYNC): ops.add("FASYNC") + if (openFlags & openflags.O_NONBLOCK): ops.add("NONBLOCK") if (openFlags & openflags.O_DIRECT): ops.add("DIRECT") if (openFlags & openflags.O_LARGEFILE): ops.add("LARGEFILE") if (openFlags & openflags.O_DIRECTORY): ops.add("DIRECTORY") - if (openFlags & openflags.O_NOFOLLOW): ops.add("NOFOLLOW") - if (openFlags & openflags.O_NOATIME): ops.add("NOATIME") if (openFlags & openflags.O_CLOEXEC): ops.add("CLOEXEC") - if (openFlags & openflags.O_SYNC): ops.add("SYNC") - if (openFlags & openflags.O_PATH): ops.add("PATH") - if (openFlags & openflags.O_TMPFILE): ops.add("TMPFILE") + if (openFlags & openflags.O_DSYNC): ops.add("DSYNC") + if (openFlags & openflags.O_SYNC): ops.add("SYNC") return ops def getTimeStr(ts): diff --git a/py3/setup.py b/py3/setup.py index 2532fba..aae4b13 100644 --- a/py3/setup.py +++ b/py3/setup.py @@ -28,7 +28,7 @@ description = ('Install SysFlow python API and utilities'), packages=['sysflow', 'sysflow.grammar'], package_data={'sysflow': ['schema.avsc']}, - install_requires=['avro-python3==1.9.1', 'avro-gen==0.3.0', 'tabulate==0.8.6', 'minio==4.0.18', 'antlr4-python3-runtime==4.7.2', 'dotty-dict==1.2.1', 'pandas==0.25.3'], + install_requires=['tabulate==0.8.6', 'minio==4.0.18', 'antlr4-python3-runtime==4.7.2', 'dotty-dict==1.2.1', 'numpy>=1.19.0', 'pandas>=0.25.3', 'frozendict==1.2', 'fastavro==0.23.6'], scripts=['utils/sysprint'], package_dir = {'': 'classes'} )