Dev

.gitignore

@@ -37,3 +37,5 @@ build
 dist
 *.egg-info
 *.antlr
+.vscode
+*.ipynb_checkpoints/

CHANGELOG.md

@@ -13,7 +13,26 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 > - **Fixed**: for any bug fixes.
 > - **Security**: in case of vulnerabilities.
 
-## [[UNRELEASED](https://github.com/sysflow-telemetry/sf-apis/compare/0.1-rc3...HEAD)]
+## [[UNRELEASED](https://github.com/sysflow-telemetry/sf-apis/compare/0.1.0-rc4...HEAD)]
+
+## [[0.1.0-rc4](https://github.com/sysflow-telemetry/sf-apis/compare/0.1-rc3...0.1.0-rc4)] - 2020-08-10
+
+### Added
+
+- Added `node.id`, `node.ip`, `proc.entry`, and `schema` attributes to query language and export APIs.
+- Added golang APIs.
+
+### Changed
+
+- Support for new Avro schema (version 2).
+- Added missing EXIT opflag to Python APIs.
+- Adding patch level to comply with semnatic versioning.
+
+### Fixed
+
+- Fixed open flags bitmaps.
+- Fixed attribute name typo when computing proc and pproc duration.
+- Fixed bug in provenance queries.
 
 ## [[0.1-rc3](https://github.com/sysflow-telemetry/sf-apis/compare/0.1-rc2...0.1-rc3)] - 2020-03-17
 

Dockerfile.sfnb

@@ -1,7 +1,6 @@
-FROM jupyter/minimal-notebook
+FROM jupyter/scipy-notebook
 
-# Install pandas, numpy, and graphviz python bindings
-RUN pip install graphviz pandas numpy matplotlib
+# Install  graphviz python bindings
 RUN conda install -y graphviz
 
 # Change user to root
@@ -14,12 +13,13 @@ RUN apt-get update -yqq && \
     apt-get clean -yqq && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/lib/apt/archive/*
 
-# Install sysflow API
-COPY py3 /build
-RUN cd /build && python setup.py install && rm -r /build
+# Copy sources
+COPY --chown=$NB_UID:$NB_UID py3 /tmp/build
 
 # Switch back to jovyan to avoid accidental container runs as root
 USER $NB_UID
 
-# Copy example scripts into container
-#COPY pynb /home/$NB_USER/work
+# Install sysflow API
+RUN cd /tmp/build && pip install . && rm -r /tmp/build
+
+

Dockerfile.sysprint

@@ -20,8 +20,10 @@ FROM registry.access.redhat.com/ubi8/ubi
 
 # Install Python environment
 RUN dnf install -y --disableplugin=subscription-manager \
+        gcc \
         python3 \
-        python3-wheel && \
+        python3-devel \
+        python3-wheel && \        
     dnf -y clean all && rm -rf /var/cache/dnf && \
     mkdir -p /usr/local/lib/python3.6/site-packages && \
     ln -s /usr/bin/easy_install-3 /usr/bin/easy_install

README.md

@@ -6,7 +6,7 @@
 
 # Supported tags and respective `Dockerfile` links
 
--	[`0.1-rc3`](https://github.com/sysflow-telemetry/sf-exporter/blob/0.1-rc3/Dockerfile), [`latest`](https://github.com/sysflow-telemetry/sf-apis/blob/master/Dockerfile)
+-	[`0.1.0-rc4`](https://github.com/sysflow-telemetry/sf-exporter/blob/0.1.0-rc4/Dockerfile), [`latest`](https://github.com/sysflow-telemetry/sf-apis/blob/master/Dockerfile)
 
 # Quick reference
 

avro/avdl/entity/file.avdl

@@ -23,7 +23,7 @@ import idl "container.avdl";
 record File {
   sysflow.type.SFObjectState state; //state of file - created, modified, reupped
   sysflow.type.FOID oid; // hash file id, container id, into 128 bit value.
-  timestamp_ms ts;
+  long ts;
   int restype; 
   string path;
   union{null, string} containerId;

avro/avdl/entity/header.avdl

@@ -19,7 +19,8 @@
 @namespace("sysflow.entity")
 protocol SysFlow {
   record SFHeader {
-     long version;
-     string exporter; 
+     long version = 2;
+     string exporter;
+     string ip = "NA";
   }
 }

avro/avdl/entity/process.avdl

@@ -24,7 +24,7 @@ record Process {
   sysflow.type.SFObjectState state; //sysflow process object state.. created, modified, reupped
   sysflow.type.OID oid; // monotonic process creation time + host PID
   union{null, sysflow.type.OID} poid;
-  timestamp_ms ts;
+  long ts;
   //int hpid;
   string exe;
   string exeArgs;
@@ -38,5 +38,6 @@ record Process {
   //int threadCount;
   //int childCount;
   union{null, string} containerId;
+  boolean entry = false; 
 }
 }

avro/avdl/event/fileevent.avdl

@@ -21,7 +21,7 @@ protocol SysFlow {
 import idl "../type/datatypes.avdl";
 record FileEvent {
   sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID
-  timestamp_ms ts;
+  long ts;
   long tid;
   int opFlags;
   sysflow.type.FOID fileOID;

avro/avdl/event/networkevent.avdl

@@ -21,7 +21,7 @@ protocol SysFlow {
 import idl "../type/datatypes.avdl";
 record NetworkEvent {
   sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID
-  timestamp_ms ts;
+  long ts;
   long tid;
   int opFlags;
   int sip;

avro/avdl/event/processevent.avdl

@@ -21,7 +21,7 @@ protocol SysFlow {
 import idl "../type/datatypes.avdl";
 record ProcessEvent {
   sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID
-  timestamp_ms ts;
+  long ts;
   long tid;
   int opFlags;
   array<string> args;

avro/avdl/flow/fileflow.avdl

@@ -21,11 +21,11 @@ protocol SysFlow {
 import idl "../type/datatypes.avdl";
 record FileFlow {
   sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID
-  timestamp_ms ts;
+  long ts;
   long tid;
   int opFlags;
   int openFlags;
-  timestamp_ms endTs;
+  long endTs;
   sysflow.type.FOID fileOID;
   int fd;
   long numRRecvOps;

avro/avdl/flow/networkflow.avdl

@@ -21,10 +21,10 @@ protocol SysFlow {
 import idl "../type/datatypes.avdl";
 record NetworkFlow {
   sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID
-  timestamp_ms ts;
+  long ts;
   long tid;
   int opFlags;
-  timestamp_ms endTs;
+  long endTs;
   int sip;
   int sport;
   int dip;

avro/avdl/flow/processflow.avdl

@@ -21,11 +21,11 @@ protocol SysFlow {
 import idl "../type/datatypes.avdl";
 record ProcessFlow {
   sysflow.type.OID procOID; // host ID + monotonic process creation time + host PID
-  timestamp_ms ts;
-  long tid;
+  long ts;
+  long numThreadsCloned;
   int opFlags;
-  timestamp_ms endTs;
-  array<string> args;
-  int ret;
+  long endTs;
+  long numThreadsExited;
+  long numCloneErrors;
 }
 }

avro/avdl/sysflow.avdl

@@ -30,6 +30,6 @@ import idl "entity/container.avdl";
 import idl "entity/file.avdl";
 
 record SysFlow {
-   union {sysflow.entity.SFHeader, sysflow.entity.Container, sysflow.entity.Process, sysflow.entity.File, sysflow.event.ProcessEvent, sysflow.flow.NetworkFlow, sysflow.flow.FileFlow, sysflow.event.FileEvent, sysflow.event.NetworkEvent} rec;
+   union {sysflow.entity.SFHeader, sysflow.entity.Container, sysflow.entity.Process, sysflow.entity.File, sysflow.event.ProcessEvent, sysflow.flow.NetworkFlow, sysflow.flow.FileFlow, sysflow.event.FileEvent, sysflow.event.NetworkEvent, sysflow.flow.ProcessFlow} rec;
 }
 }

avro/avdl/type/datatypes.avdl

@@ -19,41 +19,22 @@
 @namespace ("sysflow.type")
 
 protocol SysFlow {
-//fixed OID(16);
 
 record OID {
-  timestamp_ms createTS;
+  long createTS;
   long hpid;
 }
 
-/*record FOID {
-  long bits0;
-  long bits8;
-  long bits16;
-}*/
-
 fixed FOID(20);
 
 
 fixed ContainerID(6);
 
-/*enum EventType {
-                 CLONE, 
-                 EXEC, 
-                 EXIT 
-               }*/
 enum SFObjectState {
           CREATED, 
           MODIFIED, 
           REUP
-         }
-/*
-enum ResourceType {
-          SF_FILE,
-          SF_DIRECTORY,
-          SF_PIPE,
-          SF_UNIX
-        }*/
+}
 
 enum ContainerType {
     CT_DOCKER,
@@ -63,9 +44,9 @@ enum ContainerType {
     CT_RKT,
     CT_CUSTOM,
     CT_CRI,
-	CT_CONTAINERD,
-	CT_CRIO,
-	CT_BPM
+    CT_CONTAINERD,
+    CT_CRIO,
+    CT_BPM
 }