From afffe4a8b19d4e45d5e58ab07922205c4f223311 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 31 May 2023 14:43:06 +0200 Subject: [PATCH 01/61] Boilerplate --- .../storage/graphdb/janusgraph_provider.go | 57 +++++++++++++++++ .../storage/graphdb/janusgraph_writer.go | 64 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 pkg/kubehound/storage/graphdb/janusgraph_provider.go create mode 100644 pkg/kubehound/storage/graphdb/janusgraph_writer.go diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go new file mode 100644 index 00000000..98522530 --- /dev/null +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -0,0 +1,57 @@ +package graphdb + +import ( + "context" + + "github.com/DataDog/KubeHound/pkg/globals" + "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" + "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) + +var _ Provider = (*JanusGraphProvider)(nil) + +type JanusGraphProvider struct { + client *gremlingo.DriverRemoteConnection +} + +func NewGraphDriver(dbHost string) (*JanusGraphProvider, error) { + driver, err := gremlingo.NewDriverRemoteConnection(dbHost) + if err != nil { + return nil, err + } + + g := &JanusGraphProvider{ + client: driver, + } + + return g, nil +} + +func (jgp *JanusGraphProvider) Name() string { + return "JanusGraphProvider" +} + +func (jgp *JanusGraphProvider) HealthCheck(ctx context.Context) (bool, error) { + return true, globals.ErrNotImplemented +} + +// Raw returns a handle to the underlying provider to allow implementation specific operations e.g graph queries. +func (jgp *JanusGraphProvider) Raw() any { + return jgp.client +} + +// VertexWriter creates a new AsyncVertexWriter instance to enable asynchronous bulk inserts of vertices. +func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Vertex, opts ...WriterOption) (AsyncVertexWriter, error) { + return nil, globals.ErrNotImplemented +} + +// EdgeWriter creates a new AsyncEdgeWriter instance to enable asynchronous bulk inserts of edges. +func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Edge, opts ...WriterOption) (AsyncEdgeWriter, error) { + return nil, globals.ErrNotImplemented +} + +// Close cleans up any resources used by the Provider implementation. Provider cannot be reused after this call. +func (jgp *JanusGraphProvider) Close(ctx context.Context) error { + return globals.ErrNotImplemented +} diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go new file mode 100644 index 00000000..c1701e1e --- /dev/null +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -0,0 +1,64 @@ +package graphdb + +import ( + "context" + + "github.com/DataDog/KubeHound/pkg/globals" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) + +var _ AsyncVertexWriter = (*JanusGraphAsyncVertexWriter)(nil) + +type GremlinTraversal func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal + +type JanusGraphAsyncVertexWriter struct { + gremlin GremlinTraversal + transaction *gremlingo.Transaction + traversal *gremlingo.GraphTraversalSource + inserts []any +} + +func NewJanusGraphAsyncVertexWriter() *JanusGraphAsyncVertexWriter { + jw := JanusGraphAsyncVertexWriter{} + jw.inserts = make([]any, 0) + return &jw +} + +var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) + +type JanusGraphAsyncEdgeWriter struct { + gremlin GremlinTraversal + transaction *gremlingo.Transaction + traversal *gremlingo.GraphTraversalSource + inserts []any +} + +func NewJanusGraphAsyncEdgeWriter() *JanusGraphAsyncEdgeWriter { + jw := JanusGraphAsyncEdgeWriter{} + jw.inserts = make([]any, 0) + return &jw +} + +func (v *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { + return v.transaction.Close() +} + +func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { + return e.transaction.Close() +} + +func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { + return globals.ErrNotImplemented +} + +func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { + return globals.ErrNotImplemented +} + +func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { + return globals.ErrNotImplemented +} + +func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { + return globals.ErrNotImplemented +} From 98a0888a48e815e7666953d4d37c60c61eaad180 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 31 May 2023 15:42:14 +0200 Subject: [PATCH 02/61] Add factory --- pkg/kubehound/storage/graphdb/provider.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index aa18cc6d..7d0dcb81 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -4,7 +4,6 @@ import ( "context" "github.com/DataDog/KubeHound/pkg/config" - "github.com/DataDog/KubeHound/pkg/globals" "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" "github.com/DataDog/KubeHound/pkg/kubehound/services" @@ -65,5 +64,9 @@ type AsyncEdgeWriter interface { // Factory returns an initialized instance of a graphdb provider from the provided application config. func Factory(ctx context.Context, cfg *config.KubehoundConfig) (Provider, error) { - return nil, globals.ErrNotImplemented + provider, err := NewGraphDriver(ctx, cfg.JanusGraph.URL) + if err != nil { + return nil, err + } + return provider, nil } From 16a67780f3e8026d18b4d0dde1b22ed3199988e0 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 1 Jun 2023 12:04:52 +0200 Subject: [PATCH 03/61] config --- pkg/config/config.go | 5 +++-- pkg/config/janusgraph.go | 9 +++++++++ pkg/kubehound/storage/graphdb/janusgraph_provider.go | 6 ++++-- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 pkg/config/janusgraph.go diff --git a/pkg/config/config.go b/pkg/config/config.go index 1474700e..071186ca 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -15,8 +15,9 @@ const ( // KubehoundConfig defines the top-level application configuration for KubeHound. type KubehoundConfig struct { - Collector CollectorConfig `mapstructure:"collector"` // Collector configuration - MongoDB MongoDBConfig `mapstructure:"mongodb"` // MongoDB configuration + Collector CollectorConfig `mapstructure:"collector"` // Collector configuration + MongoDB MongoDBConfig `mapstructure:"mongodb"` // MongoDB configuration + JanusGraph JanusGraphConfig `mapstructure:"janusgraph"` // MongoDB configuration } // MustLoadDefaultConfig loads the default application configuration, treating all errors as fatal. diff --git a/pkg/config/janusgraph.go b/pkg/config/janusgraph.go new file mode 100644 index 00000000..e2105469 --- /dev/null +++ b/pkg/config/janusgraph.go @@ -0,0 +1,9 @@ +package config + +import "time" + +// MongoDBConfig configures mongodb specific parameters. +type JanusGraphConfig struct { + URL string `mapstructure:"url"` // Mongodb specific configuration + ConnectionTimeout time.Duration `mapstructure:"connection_timeout"` +} diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 98522530..5b577d13 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -15,7 +15,7 @@ type JanusGraphProvider struct { client *gremlingo.DriverRemoteConnection } -func NewGraphDriver(dbHost string) (*JanusGraphProvider, error) { +func NewGraphDriver(ctx context.Context, dbHost string) (*JanusGraphProvider, error) { driver, err := gremlingo.NewDriverRemoteConnection(dbHost) if err != nil { return nil, err @@ -53,5 +53,7 @@ func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Edge, opts // Close cleans up any resources used by the Provider implementation. Provider cannot be reused after this call. func (jgp *JanusGraphProvider) Close(ctx context.Context) error { - return globals.ErrNotImplemented + // This only logs errors + jgp.client.Close() + return nil } From 48bcdbc821148bae65a93a473ffa0f50c923c380 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 1 Jun 2023 12:51:11 +0200 Subject: [PATCH 04/61] WIP vertex writer --- .../storage/graphdb/janusgraph_writer.go | 82 +++++++++++++++---- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index c1701e1e..f95548e1 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -2,35 +2,35 @@ package graphdb import ( "context" + "sync" "github.com/DataDog/KubeHound/pkg/globals" + "github.com/DataDog/KubeHound/pkg/telemetry/log" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) var _ AsyncVertexWriter = (*JanusGraphAsyncVertexWriter)(nil) -type GremlinTraversal func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal +type GremlinTraversalVertex func(*gremlingo.GraphTraversalSource, []gremlingo.Vertex) *gremlingo.GraphTraversal +type GremlinTraversalEdge func(*gremlingo.GraphTraversalSource, []gremlingo.Edge) *gremlingo.GraphTraversal type JanusGraphAsyncVertexWriter struct { - gremlin GremlinTraversal - transaction *gremlingo.Transaction - traversal *gremlingo.GraphTraversalSource - inserts []any -} - -func NewJanusGraphAsyncVertexWriter() *JanusGraphAsyncVertexWriter { - jw := JanusGraphAsyncVertexWriter{} - jw.inserts = make([]any, 0) - return &jw + gremlin GremlinTraversalVertex + transaction *gremlingo.Transaction + traversalSource *gremlingo.GraphTraversalSource + inserts []any + consumerChan chan []gremlingo.Vertex + writingInFligth sync.WaitGroup } var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) type JanusGraphAsyncEdgeWriter struct { - gremlin GremlinTraversal - transaction *gremlingo.Transaction - traversal *gremlingo.GraphTraversalSource - inserts []any + gremlin GremlinTraversalEdge + transaction *gremlingo.Transaction + traversalSource *gremlingo.GraphTraversalSource + inserts []any + writingInFligth sync.WaitGroup } func NewJanusGraphAsyncEdgeWriter() *JanusGraphAsyncEdgeWriter { @@ -39,6 +39,56 @@ func NewJanusGraphAsyncEdgeWriter() *JanusGraphAsyncEdgeWriter { return &jw } +func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection) (*JanusGraphAsyncVertexWriter, error) { + traversal := gremlingo.Traversal_().WithRemote(drc) + tx := traversal.Tx() + gtx, err := tx.Begin() + if err != nil { + return nil, err + } + jw := JanusGraphAsyncVertexWriter{ + inserts: make([]any, 0), + transaction: tx, + traversalSource: gtx, + } + + return &jw, nil +} + +func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []gremlingo.Vertex) error { + op := jgv.gremlin(jgv.traversalSource, data) + promise := op.Iterate() + err := <-promise + if err != nil { + jgv.transaction.Rollback() + return err + } + + return nil +} + +// backgroundWriter starts a background go routine +func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { + go func() { + for { + select { + case data := <-jgv.consumerChan: + // closing the channel shoud stop the go routine + if data == nil { + return + } + err := jgv.batchWrite(ctx, data) + if err != nil { + log.I.Errorf("failed to write data in background batch writer: %w", err) + } + case <-ctx.Done(): + log.I.Info("Closed background Janus Graph worker (vertex)") + return + } + } + }() +} + func (v *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { return v.transaction.Close() } @@ -56,9 +106,11 @@ func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { } func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { + v.writingInFligth.Add(1) return globals.ErrNotImplemented } func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { + e.writingInFligth.Add(1) return globals.ErrNotImplemented } From 77711d65eb3d7e6bfa1f690ed5511277ce098c4c Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 1 Jun 2023 15:33:28 +0200 Subject: [PATCH 05/61] WIP --- .../storage/graphdb/janusgraph_writer.go | 127 +++++++++++++++--- 1 file changed, 111 insertions(+), 16 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index f95548e1..d0edaa48 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -2,9 +2,11 @@ package graphdb import ( "context" + "fmt" "sync" - "github.com/DataDog/KubeHound/pkg/globals" + "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" + "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" "github.com/DataDog/KubeHound/pkg/telemetry/log" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -18,9 +20,10 @@ type JanusGraphAsyncVertexWriter struct { gremlin GremlinTraversalVertex transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource - inserts []any + inserts []gremlingo.Vertex // Should this be gremlingo.Edge or something custom? consumerChan chan []gremlingo.Vertex writingInFligth sync.WaitGroup + batchSize int } var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) @@ -29,14 +32,27 @@ type JanusGraphAsyncEdgeWriter struct { gremlin GremlinTraversalEdge transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource - inserts []any + inserts []gremlingo.Edge // Should this be gremlingo.Edge or edge.Edge? + consumerChan chan []gremlingo.Edge writingInFligth sync.WaitGroup + batchSize int } -func NewJanusGraphAsyncEdgeWriter() *JanusGraphAsyncEdgeWriter { - jw := JanusGraphAsyncEdgeWriter{} - jw.inserts = make([]any, 0) - return &jw +func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection) (*JanusGraphAsyncEdgeWriter, error) { + traversal := gremlingo.Traversal_().WithRemote(drc) + tx := traversal.Tx() + gtx, err := tx.Begin() + if err != nil { + return nil, err + } + jw := JanusGraphAsyncEdgeWriter{ + inserts: make([]gremlingo.Edge, 0), + transaction: tx, + traversalSource: gtx, + batchSize: 1, + } + + return &jw, nil } func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection) (*JanusGraphAsyncVertexWriter, error) { @@ -47,7 +63,7 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection) (*Jan return nil, err } jw := JanusGraphAsyncVertexWriter{ - inserts: make([]any, 0), + inserts: make([]gremlingo.Vertex, 0), transaction: tx, traversalSource: gtx, } @@ -56,6 +72,9 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection) (*Jan } func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []gremlingo.Vertex) error { + jgv.writingInFligth.Add(1) + defer jgv.writingInFligth.Done() + op := jgv.gremlin(jgv.traversalSource, data) promise := op.Iterate() err := <-promise @@ -67,6 +86,21 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []g return nil } +func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []gremlingo.Edge) error { + jge.writingInFligth.Add(1) + defer jge.writingInFligth.Done() + + op := jge.gremlin(jge.traversalSource, data) + promise := op.Iterate() + err := <-promise + if err != nil { + jge.transaction.Rollback() + return err + } + + return nil +} + // backgroundWriter starts a background go routine func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { go func() { @@ -98,19 +132,80 @@ func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { } func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { - return globals.ErrNotImplemented + if v.traversalSource == nil { + return fmt.Errorf("JanusGraph traversalSource is not initialized") + } + + if len(v.inserts) == 0 { + log.I.Debugf("Skipping flush on vertex writer as no write operations left") + // we need to send something to the channel from this function whenever we don't return an error + // we cannot defer it because the go routine may last longer than the current function + // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data + v.writingInFligth.Wait() + return nil + } + + err := v.batchWrite(ctx, v.inserts) + if err != nil { + v.writingInFligth.Wait() + return err + } + + v.inserts = nil + + return nil } func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { - return globals.ErrNotImplemented + if e.traversalSource == nil { + return fmt.Errorf("JanusGraph traversalSource is not initialized") + } + + if len(e.inserts) == 0 { + log.I.Debugf("Skipping flush on edges writer as no write operations left") + // we need to send something to the channel from this function whenever we don't return an error + // we cannot defer it because the go routine may last longer than the current function + // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data + e.writingInFligth.Wait() + return nil + } + + err := e.batchWrite(ctx, e.inserts) + if err != nil { + e.writingInFligth.Wait() + return err + } + e.inserts = nil + + return nil } -func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { - v.writingInFligth.Add(1) - return globals.ErrNotImplemented +func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex vertex.Vertex) error { + if len(v.inserts) > v.batchSize { + v.consumerChan <- v.inserts + // cleanup the ops array after we have copied it to the channel + v.inserts = nil + } + converted := gremlingo.Vertex{ + Element: gremlingo.Element{ + Label: vertex.Label(), + }, + } + v.inserts = append(v.inserts, converted) + return nil } -func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { - e.writingInFligth.Add(1) - return globals.ErrNotImplemented +func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge edge.Edge) error { + if len(e.inserts) > e.batchSize { + e.consumerChan <- e.inserts + // cleanup the ops array after we have copied it to the channel + e.inserts = nil + } + converted := gremlingo.Edge{ + Element: gremlingo.Element{ + Label: edge.Label(), + }, + } + e.inserts = append(e.inserts, converted) + return nil } From 5b4b943920ee2733065a732e1772927d215a6195 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 5 Jun 2023 13:49:52 +0200 Subject: [PATCH 06/61] testing stuff --- pkg/kubehound/graph/vertex/container.go | 3 ++ pkg/kubehound/graph/vertex/identity.go | 3 ++ pkg/kubehound/graph/vertex/node.go | 1 + pkg/kubehound/graph/vertex/pod.go | 3 ++ pkg/kubehound/graph/vertex/role.go | 3 ++ pkg/kubehound/graph/vertex/volume.go | 3 ++ .../storage/graphdb/janusgraph_provider.go | 12 +++++-- .../storage/graphdb/janusgraph_writer.go | 31 +++++++++++++------ pkg/kubehound/storage/graphdb/provider.go | 4 +-- 9 files changed, 49 insertions(+), 14 deletions(-) diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 792a149f..c01e31bc 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,5 +1,7 @@ package vertex +import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + const ( containerLabel = "Container" ) @@ -7,6 +9,7 @@ const ( var _ Vertex = (*Container)(nil) type Container struct { + graph.Container } func (v Container) Label() string { diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 3243c84a..101e7057 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -1,5 +1,7 @@ package vertex +import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + const ( identityLabel = "Identity" ) @@ -7,6 +9,7 @@ const ( var _ Vertex = (*Identity)(nil) type Identity struct { + graph.Identity } func (v Identity) Label() string { diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index e35ec813..31436254 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -12,6 +12,7 @@ const ( var _ Vertex = (*Node)(nil) type Node struct { + graph.Node } func (v Node) Label() string { diff --git a/pkg/kubehound/graph/vertex/pod.go b/pkg/kubehound/graph/vertex/pod.go index fdd7d77e..c5ab8b8d 100644 --- a/pkg/kubehound/graph/vertex/pod.go +++ b/pkg/kubehound/graph/vertex/pod.go @@ -1,5 +1,7 @@ package vertex +import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + const ( podLabel = "Pod" ) @@ -7,6 +9,7 @@ const ( var _ Vertex = (*Pod)(nil) type Pod struct { + graph.Pod } func (v Pod) Label() string { diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index db101652..01c31b54 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -1,5 +1,7 @@ package vertex +import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + const ( roleLabel = "Role" ) @@ -7,6 +9,7 @@ const ( var _ Vertex = (*Role)(nil) type Role struct { + graph.Role } func (v Role) Label() string { diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index a70c5c56..e4f30f04 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -1,5 +1,7 @@ package vertex +import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + const ( volumeLabel = "Volume" ) @@ -7,6 +9,7 @@ const ( var _ Vertex = (*Volume)(nil) type Volume struct { + graph.Volume } func (v Volume) Label() string { diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 5b577d13..d9f478b8 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -43,12 +43,20 @@ func (jgp *JanusGraphProvider) Raw() any { // VertexWriter creates a new AsyncVertexWriter instance to enable asynchronous bulk inserts of vertices. func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Vertex, opts ...WriterOption) (AsyncVertexWriter, error) { - return nil, globals.ErrNotImplemented + writer, err := NewJanusGraphAsyncVertexWriter(jgp.client, opts...) + if err != nil { + return nil, err + } + return writer, nil } // EdgeWriter creates a new AsyncEdgeWriter instance to enable asynchronous bulk inserts of edges. func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Edge, opts ...WriterOption) (AsyncEdgeWriter, error) { - return nil, globals.ErrNotImplemented + writer, err := NewJanusGraphAsyncEdgeWriter(jgp.client) + if err != nil { + return nil, err + } + return writer, nil } // Close cleans up any resources used by the Provider implementation. Provider cannot be reused after this call. diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index d0edaa48..947e5304 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -2,7 +2,7 @@ package graphdb import ( "context" - "fmt" + "errors" "sync" "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" @@ -20,10 +20,10 @@ type JanusGraphAsyncVertexWriter struct { gremlin GremlinTraversalVertex transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource - inserts []gremlingo.Vertex // Should this be gremlingo.Edge or something custom? + inserts []gremlingo.Vertex // Should this be gremlingo.Edge or vertex.Vertex? consumerChan chan []gremlingo.Vertex writingInFligth sync.WaitGroup - batchSize int + batchSize int // Shouldn't this be "per vertex types" ? } var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) @@ -35,10 +35,15 @@ type JanusGraphAsyncEdgeWriter struct { inserts []gremlingo.Edge // Should this be gremlingo.Edge or edge.Edge? consumerChan chan []gremlingo.Edge writingInFligth sync.WaitGroup - batchSize int + batchSize int // Shouldn't this be "per edge types" ? } -func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection) (*JanusGraphAsyncEdgeWriter, error) { +func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { + options := &writerOptions{} + for _, opt := range opts { + opt(options) + } + traversal := gremlingo.Traversal_().WithRemote(drc) tx := traversal.Tx() gtx, err := tx.Begin() @@ -55,7 +60,12 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection) (*Janus return &jw, nil } -func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection) (*JanusGraphAsyncVertexWriter, error) { +func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { + options := &writerOptions{} + for _, opt := range opts { + opt(options) + } + traversal := gremlingo.Traversal_().WithRemote(drc) tx := traversal.Tx() gtx, err := tx.Begin() @@ -113,7 +123,7 @@ func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { } err := jgv.batchWrite(ctx, data) if err != nil { - log.I.Errorf("failed to write data in background batch writer: %w", err) + log.I.Errorf("failed to write data in background batch writer: %v", err) } case <-ctx.Done(): log.I.Info("Closed background Janus Graph worker (vertex)") @@ -133,7 +143,7 @@ func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { if v.traversalSource == nil { - return fmt.Errorf("JanusGraph traversalSource is not initialized") + return errors.New("JanusGraph traversalSource is not initialized") } if len(v.inserts) == 0 { @@ -150,7 +160,6 @@ func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { v.writingInFligth.Wait() return err } - v.inserts = nil return nil @@ -158,7 +167,7 @@ func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { if e.traversalSource == nil { - return fmt.Errorf("JanusGraph traversalSource is not initialized") + return errors.New("JanusGraph traversalSource is not initialized") } if len(e.inserts) == 0 { @@ -188,6 +197,7 @@ func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex vertex.V } converted := gremlingo.Vertex{ Element: gremlingo.Element{ + Id: vertex.Label(), //FIXME, not sure what should be added here Label: vertex.Label(), }, } @@ -203,6 +213,7 @@ func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge edge.Edge) e } converted := gremlingo.Edge{ Element: gremlingo.Element{ + Id: edge.Label(), //FIXME, not sure what should be added here Label: edge.Label(), }, } diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index 2bae817a..5f2d7641 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -48,7 +48,7 @@ type AsyncVertexWriter interface { WriterBase // Queue add a vertex model to an asynchronous write queue. Non-blocking. - Queue(ctx context.Context, v any) error + Queue(ctx context.Context, v vertex.Vertex) error } // AsyncEdgeWriter defines the interface for writer clients to queue aysnchronous, batched writes of edges to the graphdb. @@ -58,7 +58,7 @@ type AsyncEdgeWriter interface { WriterBase // Queue add an edge model to an asynchronous write queue. Non-blocking. - Queue(ctx context.Context, e any) error + Queue(ctx context.Context, e edge.Edge) error } // Factory returns an initialized instance of a graphdb provider from the provided application config. From 27b3cd9f32700400b533b1528a5f7fa068e8b6e5 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Tue, 6 Jun 2023 19:11:30 +0200 Subject: [PATCH 07/61] rollback any instead of vertex.Vertex --- pkg/kubehound/storage/graphdb/janusgraph_writer.go | 6 ++---- pkg/kubehound/storage/graphdb/provider.go | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 947e5304..fe5a29a4 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -5,8 +5,6 @@ import ( "errors" "sync" - "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" - "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" "github.com/DataDog/KubeHound/pkg/telemetry/log" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -189,7 +187,7 @@ func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { return nil } -func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex vertex.Vertex) error { +func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { if len(v.inserts) > v.batchSize { v.consumerChan <- v.inserts // cleanup the ops array after we have copied it to the channel @@ -205,7 +203,7 @@ func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex vertex.V return nil } -func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge edge.Edge) error { +func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { if len(e.inserts) > e.batchSize { e.consumerChan <- e.inserts // cleanup the ops array after we have copied it to the channel diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index ea2e25b2..0438e053 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -48,7 +48,7 @@ type AsyncVertexWriter interface { WriterBase // Queue add a vertex model to an asynchronous write queue. Non-blocking. - Queue(ctx context.Context, v vertex.Vertex) error + Queue(ctx context.Context, v any) error } // AsyncEdgeWriter defines the interface for writer clients to queue aysnchronous, batched writes of edges to the graphdb. @@ -58,7 +58,7 @@ type AsyncEdgeWriter interface { WriterBase // Queue add an edge model to an asynchronous write queue. Non-blocking. - Queue(ctx context.Context, e edge.Edge) error + Queue(ctx context.Context, e any) error } // Factory returns an initialized instance of a graphdb provider from the provided application config. From 04ce2c67c162f2da3c7056bc2b9817b83cdbb169 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 7 Jun 2023 10:32:47 +0200 Subject: [PATCH 08/61] Updates --- pkg/config/config.go | 2 +- pkg/kubehound/graph/vertex/container.go | 25 ++++++++++++++++++++++--- pkg/kubehound/graph/vertex/identity.go | 3 --- pkg/kubehound/graph/vertex/node.go | 1 - pkg/kubehound/graph/vertex/pod.go | 3 --- pkg/kubehound/graph/vertex/role.go | 3 --- pkg/kubehound/graph/vertex/volume.go | 3 --- 7 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 22d80638..63ec0c03 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,7 +18,7 @@ const ( type KubehoundConfig struct { Collector CollectorConfig `mapstructure:"collector"` // Collector configuration MongoDB MongoDBConfig `mapstructure:"mongodb"` // MongoDB configuration - JanusGraph JanusGraphConfig `mapstructure:"janusgraph"` // MongoDB configuration + JanusGraph JanusGraphConfig `mapstructure:"janusgraph"` // JanusGraph configuration Telemetry TelemetryConfig `mapstructure:"telemetry"` // telemetry configuration, contains statsd and other sub structures } diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 88a6fdcb..b8acc59a 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,6 +1,6 @@ package vertex -import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" +import gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" const ( containerLabel = "Container" @@ -9,7 +9,6 @@ const ( var _ Builder = (*Container)(nil) type Container struct { - graph.Container } func (v Container) Label() string { @@ -21,5 +20,25 @@ func (v Container) BatchSize() int { } func (v Container) Traversal() VertexTraversal { - return nil + return (func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { + return g.Inject(inserts).Unfold().As("c"). + AddV("Container"). + Property("storeId", gremlingo.T__.Select("c").Select("store_id")). + Property("name", gremlingo.T__.Select("c").Select("name")). + Property("image", gremlingo.T__.Select("c").Select("image")). + Property("capabilities", gremlingo.T__.Select("c").Select("capabilities")). + Property("command", gremlingo.T__.Select("c").Select("command")). + Property("capabilities", gremlingo.T__.Select("c").Select("capabilities")). + Property("privileged", gremlingo.T__.Select("c").Select("privileged")). + Property("privesc", gremlingo.T__.Select("c").Select("privesc")). + Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). + Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). + Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). + Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). + Property("ports", gremlingo.T__.Select("c").Select("ports")). + Property("pod", gremlingo.T__.Select("c").Select("pod")). + Property("node", gremlingo.T__.Select("c").Select("node")). + Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + Property("critical", gremlingo.T__.Select("c").Select("critical")) + }) } diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 6e9d4bc1..686d7901 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -1,7 +1,5 @@ package vertex -import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - const ( identityLabel = "Identity" ) @@ -9,7 +7,6 @@ const ( var _ Builder = (*Identity)(nil) type Identity struct { - graph.Identity } func (v Identity) Label() string { diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index ebb804f7..7c831ebd 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -12,7 +12,6 @@ const ( var _ Builder = (*Node)(nil) type Node struct { - graph.Node } func (v Node) Label() string { diff --git a/pkg/kubehound/graph/vertex/pod.go b/pkg/kubehound/graph/vertex/pod.go index cc102db5..4e8d1cda 100644 --- a/pkg/kubehound/graph/vertex/pod.go +++ b/pkg/kubehound/graph/vertex/pod.go @@ -1,7 +1,5 @@ package vertex -import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - const ( podLabel = "Pod" ) @@ -9,7 +7,6 @@ const ( var _ Builder = (*Pod)(nil) type Pod struct { - graph.Pod } func (v Pod) Label() string { diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index 279298a3..8663ec25 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -1,7 +1,5 @@ package vertex -import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - const ( roleLabel = "Role" ) @@ -9,7 +7,6 @@ const ( var _ Builder = (*Role)(nil) type Role struct { - graph.Role } func (v Role) Label() string { diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index b5444833..f73d1381 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -1,7 +1,5 @@ package vertex -import "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - const ( volumeLabel = "Volume" ) @@ -9,7 +7,6 @@ const ( var _ Builder = (*Volume)(nil) type Volume struct { - graph.Volume } func (v Volume) Label() string { From f16332efdd26411a72b8b0972bcb707889b38d52 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 7 Jun 2023 10:37:10 +0200 Subject: [PATCH 09/61] remove () --- pkg/kubehound/graph/vertex/container.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index b8acc59a..5e489b72 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -20,7 +20,7 @@ func (v Container) BatchSize() int { } func (v Container) Traversal() VertexTraversal { - return (func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { + return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { return g.Inject(inserts).Unfold().As("c"). AddV("Container"). Property("storeId", gremlingo.T__.Select("c").Select("store_id")). @@ -40,5 +40,5 @@ func (v Container) Traversal() VertexTraversal { Property("node", gremlingo.T__.Select("c").Select("node")). Property("compromised", gremlingo.T__.Select("c").Select("compromised")). Property("critical", gremlingo.T__.Select("c").Select("critical")) - }) + } } From cdbb679d8a615922cc6b12f1f53e3d513018ea1d Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 7 Jun 2023 10:43:33 +0200 Subject: [PATCH 10/61] Identity --- pkg/kubehound/graph/vertex/identity.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 686d7901..5b5ed6b9 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -1,5 +1,7 @@ package vertex +import gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + const ( identityLabel = "Identity" ) @@ -18,5 +20,13 @@ func (v Identity) BatchSize() int { } func (v Identity) Traversal() VertexTraversal { - return nil + return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { + return g.Inject(inserts).Unfold().As("c"). + AddV("Identity"). + Property("storeId", gremlingo.T__.Select("c").Select("store_id")). + Property("name", gremlingo.T__.Select("c").Select("name")). + Property("isNamespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + Property("namespace", gremlingo.T__.Select("c").Select("namespace")). + Property("type", gremlingo.T__.Select("c").Select("type")) + } } From 4acf5c490cc872b9a3ae45eb43f7b0629b851798 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 7 Jun 2023 14:04:00 +0200 Subject: [PATCH 11/61] :i-have-no-idea-what-im-doing: --- configs/etc/kubehound.yaml | 2 + pkg/kubehound/graph/vertex/container.go | 14 ++- pkg/kubehound/graph/vertex/container_test.go | 94 ++++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 pkg/kubehound/graph/vertex/container_test.go diff --git a/configs/etc/kubehound.yaml b/configs/etc/kubehound.yaml index ebc6f3ff..00e415a9 100644 --- a/configs/etc/kubehound.yaml +++ b/configs/etc/kubehound.yaml @@ -6,6 +6,8 @@ collector: rate_limit_per_second: 100 mongodb: url: "mongodb://localhost:27017" +janusgraph: + url: "localhost:8182" telemetry: statsd: url: "127.0.0.1:8125" \ No newline at end of file diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 5e489b72..a38dc206 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,6 +1,8 @@ package vertex -import gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +import ( + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) const ( containerLabel = "Container" @@ -21,6 +23,16 @@ func (v Container) BatchSize() int { func (v Container) Traversal() VertexTraversal { return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { + // g = g.GetGraphTraversal() + + // for _, insert := range inserts { + // i := insert.(graph.Container) + // g.AddV(v.Label()). + // Property("id", i.StoreId). + // Property("name", i.Name) + // } + + // return g.GetGraphTraversal() return g.Inject(inserts).Unfold().As("c"). AddV("Container"). Property("storeId", gremlingo.T__.Select("c").Select("store_id")). diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go new file mode 100644 index 00000000..292b1285 --- /dev/null +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -0,0 +1,94 @@ +package vertex + +import ( + "testing" + + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + "github.com/stretchr/testify/assert" +) + +func TestContainer_Traversal(t *testing.T) { + tests := []struct { + name string + v Container + want VertexTraversal + }{ + { + name: "Add containers in JanusGraph", + v: Container{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dbHost := "ws://localhost:8182/gremlin" + driver, err := gremlingo.NewDriverRemoteConnection(dbHost) + assert.NoError(t, err) + + traversal := gremlingo.Traversal_().WithRemote(driver) + tx := traversal.Tx() + g, err := tx.Begin() + assert.NoError(t, err) + + v := Container{} + // We set the values to all field with non default values so we are sure all are correctly propagated. + insert := graph.Container{ + StoreId: "test id", + Name: "test name", + Image: "image", + Command: []string{"/usr/bin/sleep"}, + Args: []string{"600"}, + Capabilities: []string{"CAPABILITY"}, + Privileged: true, + PrivEsc: true, + HostPID: true, + HostPath: true, + HostIPC: true, + HostNetwork: true, + RunAsUser: 1234, + Ports: []int{1337}, + Pod: "test pod", + Node: "test node", + Compromised: 1, + Critical: true, + } + vertexTraversal := v.Traversal() + _ = vertexTraversal(g, []TraversalInput{insert}) + + err = tx.Commit() + assert.NoError(t, err) + // err = tx.Close() + // assert.NoError(t, err) + + driver, err = gremlingo.NewDriverRemoteConnection(dbHost) + assert.NoError(t, err) + + traversal = gremlingo.Traversal_().WithRemote(driver) + tx = traversal.Tx() + g, err = tx.Begin() + + assert.NoError(t, err) + test := g.V().HasLabel(v.Label()).Properties() + res, err := test.Traversal.GetResultSet() + assert.NoError(t, err) + data, err := res.All() + + for _, d := range data { + vertex, err := d.GetVertex() + assert.NoError(t, err) + t.Errorf("vertex: %+v", vertex) + t.Errorf("vertex elem: %+v", vertex.Element) + t.Errorf("vertex id: %+v", vertex.Id) + t.Errorf("vertex label: %+v", vertex.Label) + } + + err = tx.Commit() + assert.NoError(t, err) + // err = tx.Close() + // assert.NoError(t, err) + t.Errorf("%+v", res) + t.Errorf("%+v", data) + }) + } +} From 05aa3d1048c1581c88871f9d9a2103ffd63683f9 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 7 Jun 2023 15:53:44 +0200 Subject: [PATCH 12/61] Add mapstructure --- pkg/kubehound/models/graph/models.go | 112 +++++++++++++-------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/pkg/kubehound/models/graph/models.go b/pkg/kubehound/models/graph/models.go index 75b4c50e..63d2bc2c 100644 --- a/pkg/kubehound/models/graph/models.go +++ b/pkg/kubehound/models/graph/models.go @@ -9,76 +9,76 @@ const ( ) type Container struct { - StoreId string `json:"store_id"` - Name string `json:"name"` - Image string `json:"image"` - Command []string `json:"command"` - Args []string `json:"args"` - Capabilities []string `json:"capabilities"` - Privileged bool `json:"privileged"` - PrivEsc bool `json:"privesc"` - HostPID bool `json:"hostPid"` - HostPath bool `json:"hostPath"` - HostIPC bool `json:"hostIpc"` - HostNetwork bool `json:"hostNetwork"` - RunAsUser int64 `json:"runAsUser"` - Ports []int `json:"ports"` - Pod string `json:"pod"` - Node string `json:"node"` - Compromised CompromiseType `json:"compromised,omitempty"` - Critical bool `json:"critical,omitempty"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` + Image string `json:"image" mapstructure:"image"` + Command []string `json:"command" mapstructure:"command"` + Args []string `json:"args" mapstructure:"args"` + Capabilities []string `json:"capabilities" mapstructure:"capabilities"` + Privileged bool `json:"privileged" mapstructure:"privileged"` + PrivEsc bool `json:"privesc" mapstructure:"privesc"` + HostPID bool `json:"hostPid" mapstructure:"hostPid"` + HostPath bool `json:"hostPath" mapstructure:"hostPath"` + HostIPC bool `json:"hostIpc" mapstructure:"hostIpc"` + HostNetwork bool `json:"hostNetwork" mapstructure:"hostNetwork"` + RunAsUser int64 `json:"runAsUser" mapstructure:"runAsUser"` + Ports []int `json:"ports" mapstructure:"ports"` + Pod string `json:"pod" mapstructure:"pod"` + Node string `json:"node" mapstructure:"node"` + Compromised CompromiseType `json:"compromised,omitempty" mapstructure:"compromised"` + Critical bool `json:"critical,omitempty" mapstructure:"critical"` } type Group struct { - StoreId string `json:"store_id"` - Name string `json:"name"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` } type Identity struct { - StoreId string `json:"store_id"` - Name string `json:"name"` - IsNamespaced bool `json:"is_namespaced"` - Namespace string `json:"namespace"` - Type string `json:"type"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` + IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + Namespace string `json:"namespace" mapstructure:"namespace"` + Type string `json:"type" mapstructure:"type"` } type Node struct { - StoreId string `json:"store_id"` - Name string `json:"name"` - IsNamespaced bool `json:"is_namespaced"` - Namespace string `json:"namespace"` - Compromised CompromiseType `json:"compromised,omitempty"` - Critical bool `json:"critical,omitempty"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` + IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + Namespace string `json:"namespace" mapstructure:"namespace"` + Compromised CompromiseType `json:"compromised,omitempty" mapstructure:"compromised"` + Critical bool `json:"critical,omitempty" mapstructure:"critical"` } type Pod struct { - StoreId string `json:"store_id"` - Name string `json:"name"` - IsNamespaced bool `json:"is_namespaced"` - Namespace string `json:"namespace"` - SharedProcessNamespace bool `json:"sharedProcessNamespace"` - ServiceAccount string `json:"serviceAccount"` - Node string `json:"node"` - Compromised CompromiseType `json:"compromised,omitempty"` - Critical bool `json:"critical,omitempty"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` + IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + Namespace string `json:"namespace" mapstructure:"namespace"` + SharedProcessNamespace bool `json:"sharedProcessNamespace" mapstructure:"sharedProcessNamespace"` + ServiceAccount string `json:"serviceAccount" mapstructure:"serviceAccount"` + Node string `json:"node" mapstructure:"node"` + Compromised CompromiseType `json:"compromised,omitempty" mapstructure:"compromised"` + Critical bool `json:"critical,omitempty" mapstructure:"critical"` } type Role struct { - StoreId string `json:"store_id"` - Name string `json:"name"` - IsNamespaced bool `json:"is_namespaced"` - Namespace string `json:"namespace"` - Rules []string `json:"rules"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` + IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + Namespace string `json:"namespace" mapstructure:"namespace"` + Rules []string `json:"rules" mapstructure:"rules"` } type Token struct { - StoreId string `json:"store_id"` - Name string `json:"name"` - Type string `json:"type"` - Identity string `json:"identity"` - Path string `json:"path"` - Compromised CompromiseType `json:"compromised,omitempty"` - Critical bool `json:"critical,omitempty"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` + Type string `json:"type" mapstructure:"type"` + Identity string `json:"identity" mapstructure:"identity"` + Path string `json:"path" mapstructure:"path"` + Compromised CompromiseType `json:"compromised,omitempty" mapstructure:"compromised"` + Critical bool `json:"critical,omitempty" mapstructure:"critical"` } const ( @@ -87,8 +87,8 @@ const ( ) type Volume struct { - StoreId string `json:"store_id"` - Name string `json:"name"` - Type string `json:"type"` - Path string `json:"path"` + StoreId string `json:"store_id" mapstructure:"store_id"` + Name string `json:"name" mapstructure:"name"` + Type string `json:"type" mapstructure:"type"` + Path string `json:"path" mapstructure:"path"` } From 6f98d62aee385783dbffa70de0ad82f335c9f483 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Wed, 7 Jun 2023 19:04:27 +0200 Subject: [PATCH 13/61] '{code:244 message:Property value [[failed_to_parse_command]] is of type class java.util.ArrayList is not supported --- pkg/kubehound/graph/edge/mongo.go | 22 +--- pkg/kubehound/graph/vertex/container.go | 100 ++++++++++++++----- pkg/kubehound/graph/vertex/container_test.go | 51 +++++----- pkg/utils/structomap.go | 21 ++++ 4 files changed, 122 insertions(+), 72 deletions(-) create mode 100644 pkg/utils/structomap.go diff --git a/pkg/kubehound/graph/edge/mongo.go b/pkg/kubehound/graph/edge/mongo.go index 0017bedc..84742d2e 100644 --- a/pkg/kubehound/graph/edge/mongo.go +++ b/pkg/kubehound/graph/edge/mongo.go @@ -2,11 +2,11 @@ package edge import ( "context" - "encoding/json" "fmt" "github.com/DataDog/KubeHound/pkg/kubehound/storage/storedb" "github.com/DataDog/KubeHound/pkg/telemetry/log" + "github.com/DataDog/KubeHound/pkg/utils" "go.mongodb.org/mongo-driver/mongo" ) @@ -27,7 +27,7 @@ func MongoProcessor[T any](_ context.Context, entry DataContainer) (map[string]a return nil, fmt.Errorf("invalid type passed to processor: %T", entry) } - processed, err := structToMap(typed) + processed, err := utils.StructToMap(typed) if err != nil { return nil, err } @@ -54,21 +54,3 @@ func MongoCursorHandler[T any](ctx context.Context, cur *mongo.Cursor, return complete(ctx) } - -// structToMap transforms a struct to a map to be consumed by a mongoDB AsyncWriter implementation. -// TODO: review implementation... surely there's a better way? -func structToMap(in any) (map[string]any, error) { - var res map[string]any - - tmp, err := json.Marshal(in) - if err != nil { - return nil, err - } - - err = json.Unmarshal(tmp, &res) - if err != nil { - return nil, err - } - - return res, nil -} diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index a38dc206..6764f138 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,6 +1,8 @@ package vertex import ( + "fmt" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -25,32 +27,80 @@ func (v Container) Traversal() VertexTraversal { return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { // g = g.GetGraphTraversal() - // for _, insert := range inserts { - // i := insert.(graph.Container) - // g.AddV(v.Label()). - // Property("id", i.StoreId). - // Property("name", i.Name) - // } + for _, insert := range inserts { + i := insert.(map[string]any) + command, ok := i["command"].([]string) + if !ok { + command = []string{"failed_to_parse_command"} + } + + args, ok := i["args"].([]string) + if !ok { + args = []string{"failed_to_parse_args"} + } + + capabilities, ok := i["capabilities"].([]string) + if !ok { + capabilities = []string{"failed_to_parse_capabilities"} + } + ports, ok := i["ports"].([]int) + if !ok { + ports = []int{0} + } + promise := g.AddV(v.Label()). + Property("storeId", i["storeId"]). + Property("name", i["name"]). + Property("image", i["image"]). + Property([]interface{}{gremlingo.Cardinality.Set, "command", command}...). + Property([]interface{}{gremlingo.Cardinality.Set, "args", args}...). + Property([]interface{}{gremlingo.Cardinality.Set, "capabilities", capabilities}...). + Property("privileged", i["privileged"]). + Property("privesc", i["privesc"]). + Property("hostPid", i["hostPid"]). + Property("hostPath", i["hostPath"]). + Property("hostNetwork", i["hostNetwork"]). + Property("runAsUser", i["runAsUser"]). + Property([]interface{}{gremlingo.Cardinality.Set, "ports", ports}...). + Property("pod", i["pod"]). + Property("node", i["node"]). + Property("compromised", i["compromised"]). + Property("critical", i["critical"]). + Iterate() + err := <-promise + if err != nil { + fmt.Printf("promise err: %v\n", err) + } + } + return g.GetGraphTraversal() + + // I have no idea how I can convert the arrays in the inserts to multiple values with cardinality set.... + // promise := g.Inject(inserts).Unfold().As("c"). + // AddV(v.Label()). + // Property("storeId", gremlingo.T__.Select("c").Select("store_id")). + // Property("name", gremlingo.T__.Select("c").Select("name")). + // Property("image", gremlingo.T__.Select("c").Select("image")). + // Property(gremlingo.Cardinality.Set, "command", gremlingo.T__.Select("c").Select("command")). + // Property(gremlingo.Cardinality.Set, "args", gremlingo.T__.Select("c").Select("args")). + // Property(gremlingo.Cardinality.Set, "capabilities", gremlingo.T__.Select("c").Select("capabilities")). + // Property("privileged", gremlingo.T__.Select("c").Select("privileged")). + // Property("privesc", gremlingo.T__.Select("c").Select("privesc")). + // Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). + // Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). + // Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). + // Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). + // Property(gremlingo.Cardinality.Set, "ports", gremlingo.T__.Select("c").Select("ports")). + // Property("pod", gremlingo.T__.Select("c").Select("pod")). + // Property("node", gremlingo.T__.Select("c").Select("node")). + // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + // Property("critical", gremlingo.T__.Select("c").Select("critical")). + // Iterate() + + // // The returned promised is a go channel to wait for all submitted steps to finish execution and return error. + // err := <-promise + // if err != nil { + // fmt.Println(err) + // } // return g.GetGraphTraversal() - return g.Inject(inserts).Unfold().As("c"). - AddV("Container"). - Property("storeId", gremlingo.T__.Select("c").Select("store_id")). - Property("name", gremlingo.T__.Select("c").Select("name")). - Property("image", gremlingo.T__.Select("c").Select("image")). - Property("capabilities", gremlingo.T__.Select("c").Select("capabilities")). - Property("command", gremlingo.T__.Select("c").Select("command")). - Property("capabilities", gremlingo.T__.Select("c").Select("capabilities")). - Property("privileged", gremlingo.T__.Select("c").Select("privileged")). - Property("privesc", gremlingo.T__.Select("c").Select("privesc")). - Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). - Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). - Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). - Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). - Property("ports", gremlingo.T__.Select("c").Select("ports")). - Property("pod", gremlingo.T__.Select("c").Select("pod")). - Property("node", gremlingo.T__.Select("c").Select("node")). - Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - Property("critical", gremlingo.T__.Select("c").Select("critical")) } } diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go index 292b1285..cd823270 100644 --- a/pkg/kubehound/graph/vertex/container_test.go +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -4,7 +4,9 @@ import ( "testing" "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + "github.com/DataDog/KubeHound/pkg/utils" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/assert" ) @@ -26,20 +28,20 @@ func TestContainer_Traversal(t *testing.T) { driver, err := gremlingo.NewDriverRemoteConnection(dbHost) assert.NoError(t, err) - traversal := gremlingo.Traversal_().WithRemote(driver) - tx := traversal.Tx() - g, err := tx.Begin() + g := gremlingo.Traversal_().WithRemote(driver) + // tx := traversal.Tx() + // g, err := tx.Begin() assert.NoError(t, err) v := Container{} // We set the values to all field with non default values so we are sure all are correctly propagated. - insert := graph.Container{ + insert, err := utils.StructToMap(graph.Container{ StoreId: "test id", Name: "test name", Image: "image", Command: []string{"/usr/bin/sleep"}, Args: []string{"600"}, - Capabilities: []string{"CAPABILITY"}, + Capabilities: []string{"NET_CAP_ADMIN", "NET_RAW_ADMINaa"}, Privileged: true, PrivEsc: true, HostPID: true, @@ -52,43 +54,38 @@ func TestContainer_Traversal(t *testing.T) { Node: "test node", Compromised: 1, Critical: true, - } + }) + assert.NoError(t, err) + vertexTraversal := v.Traversal() - _ = vertexTraversal(g, []TraversalInput{insert}) + inserts := []TraversalInput{insert} + t.Errorf("inserts: %v", inserts) + _ = vertexTraversal(g, inserts) - err = tx.Commit() - assert.NoError(t, err) + // err = tx.Commit() + // assert.NoError(t, err) // err = tx.Close() // assert.NoError(t, err) driver, err = gremlingo.NewDriverRemoteConnection(dbHost) assert.NoError(t, err) - traversal = gremlingo.Traversal_().WithRemote(driver) - tx = traversal.Tx() - g, err = tx.Begin() + g = gremlingo.Traversal_().WithRemote(driver) + // tx = traversal.Tx() + // g, err = tx.Begin() assert.NoError(t, err) - test := g.V().HasLabel(v.Label()).Properties() + test := g.V().HasLabel(v.Label()).ValueMap() res, err := test.Traversal.GetResultSet() assert.NoError(t, err) data, err := res.All() - for _, d := range data { - vertex, err := d.GetVertex() - assert.NoError(t, err) - t.Errorf("vertex: %+v", vertex) - t.Errorf("vertex elem: %+v", vertex.Element) - t.Errorf("vertex id: %+v", vertex.Id) - t.Errorf("vertex label: %+v", vertex.Label) + thefuck := d.GetInterface() + t.Errorf("thefuck: %+v", thefuck) + container := graph.Container{} + mapstructure.Decode(thefuck, &container) + t.Errorf("container: %+v", container) } - - err = tx.Commit() - assert.NoError(t, err) - // err = tx.Close() - // assert.NoError(t, err) - t.Errorf("%+v", res) - t.Errorf("%+v", data) }) } } diff --git a/pkg/utils/structomap.go b/pkg/utils/structomap.go new file mode 100644 index 00000000..8d32ff42 --- /dev/null +++ b/pkg/utils/structomap.go @@ -0,0 +1,21 @@ +package utils + +import "encoding/json" + +// structToMap transforms a struct to a map to be consumed by a mongoDB AsyncWriter implementation. +// TODO: review implementation... surely there's a better way? +func StructToMap(in any) (map[string]any, error) { + var res map[string]any + + tmp, err := json.Marshal(in) + if err != nil { + return nil, err + } + + err = json.Unmarshal(tmp, &res) + if err != nil { + return nil, err + } + + return res, nil +} From 057d7fa970642af522049b7188eadff1ba729b16 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 09:15:56 +0200 Subject: [PATCH 14/61] add vertices --- pkg/kubehound/graph/vertex/container.go | 124 ++++++++----------- pkg/kubehound/graph/vertex/container_test.go | 90 +++++++------- pkg/kubehound/graph/vertex/node.go | 23 ++-- pkg/kubehound/graph/vertex/pod.go | 20 ++- pkg/kubehound/graph/vertex/role.go | 16 ++- pkg/kubehound/graph/vertex/volume.go | 15 ++- pkg/utils/interfaces.go | 26 ++++ 7 files changed, 185 insertions(+), 129 deletions(-) create mode 100644 pkg/utils/interfaces.go diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 6764f138..cfeac48a 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,8 +1,6 @@ package vertex import ( - "fmt" - gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -27,80 +25,64 @@ func (v Container) Traversal() VertexTraversal { return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { // g = g.GetGraphTraversal() - for _, insert := range inserts { - i := insert.(map[string]any) - command, ok := i["command"].([]string) - if !ok { - command = []string{"failed_to_parse_command"} - } + // for _, insert := range inserts { + // i := insert.(map[string]any) + // // command := utils.ToSliceOfAny(i["command"].([]any)) + // // args := utils.ToSliceOfAny(i["args"].([]any)) + // // capabilities := utils.ToSliceOfAny(i["capabilities"].([]any)) - args, ok := i["args"].([]string) - if !ok { - args = []string{"failed_to_parse_args"} - } + // traversal := g.AddV(v.Label()). + // Property("storeId", i["storeId"]). + // Property("name", i["name"]). + // Property("image", i["image"]). + // Property("privileged", i["privileged"]). + // Property("privesc", i["privesc"]). + // Property("hostPid", i["hostPid"]). + // Property("hostPath", i["hostPath"]). + // Property("hostNetwork", i["hostNetwork"]). + // Property("runAsUser", i["runAsUser"]). + // Property("pod", i["pod"]). + // Property("node", i["node"]). + // Property("compromised", i["compromised"]). + // Property("critical", i["critical"]) - capabilities, ok := i["capabilities"].([]string) - if !ok { - capabilities = []string{"failed_to_parse_capabilities"} - } - ports, ok := i["ports"].([]int) - if !ok { - ports = []int{0} - } + // // for _, cmd := range command { + // // traversal = traversal.Property(gremlingo.Cardinality.Set, "command", cmd) + // // } + // // for _, arg := range args { + // // traversal = traversal.Property(gremlingo.Cardinality.Set, "args", arg) + // // } + // // for _, cap := range capabilities { + // // traversal = traversal.Property(gremlingo.Cardinality.Set, "capabilities", cap) + // // } + // // for _, port := range ports { + // // traversal = traversal.Property(gremlingo.Cardinality.Set, "ports", port) + // // } + // } - promise := g.AddV(v.Label()). - Property("storeId", i["storeId"]). - Property("name", i["name"]). - Property("image", i["image"]). - Property([]interface{}{gremlingo.Cardinality.Set, "command", command}...). - Property([]interface{}{gremlingo.Cardinality.Set, "args", args}...). - Property([]interface{}{gremlingo.Cardinality.Set, "capabilities", capabilities}...). - Property("privileged", i["privileged"]). - Property("privesc", i["privesc"]). - Property("hostPid", i["hostPid"]). - Property("hostPath", i["hostPath"]). - Property("hostNetwork", i["hostNetwork"]). - Property("runAsUser", i["runAsUser"]). - Property([]interface{}{gremlingo.Cardinality.Set, "ports", ports}...). - Property("pod", i["pod"]). - Property("node", i["node"]). - Property("compromised", i["compromised"]). - Property("critical", i["critical"]). - Iterate() - err := <-promise - if err != nil { - fmt.Printf("promise err: %v\n", err) - } - } - return g.GetGraphTraversal() + // return g.GetGraphTraversal() // I have no idea how I can convert the arrays in the inserts to multiple values with cardinality set.... - // promise := g.Inject(inserts).Unfold().As("c"). - // AddV(v.Label()). - // Property("storeId", gremlingo.T__.Select("c").Select("store_id")). - // Property("name", gremlingo.T__.Select("c").Select("name")). - // Property("image", gremlingo.T__.Select("c").Select("image")). - // Property(gremlingo.Cardinality.Set, "command", gremlingo.T__.Select("c").Select("command")). - // Property(gremlingo.Cardinality.Set, "args", gremlingo.T__.Select("c").Select("args")). - // Property(gremlingo.Cardinality.Set, "capabilities", gremlingo.T__.Select("c").Select("capabilities")). - // Property("privileged", gremlingo.T__.Select("c").Select("privileged")). - // Property("privesc", gremlingo.T__.Select("c").Select("privesc")). - // Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). - // Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). - // Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). - // Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). - // Property(gremlingo.Cardinality.Set, "ports", gremlingo.T__.Select("c").Select("ports")). - // Property("pod", gremlingo.T__.Select("c").Select("pod")). - // Property("node", gremlingo.T__.Select("c").Select("node")). - // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - // Property("critical", gremlingo.T__.Select("c").Select("critical")). - // Iterate() + traversal := g.Inject(inserts).Unfold().As("c"). + AddV(v.Label()). + Property("storeId", gremlingo.T__.Select("c").Select("store_id")). + Property("name", gremlingo.T__.Select("c").Select("name")). + Property("image", gremlingo.T__.Select("c").Select("image")). + // Property(gremlingo.Cardinality.Set, "command", gremlingo.T__.Select("c").Select("command")). + // Property(gremlingo.Cardinality.Set, "args", gremlingo.T__.Select("c").Select("args")). + // Property(gremlingo.Cardinality.Set, "capabilities", gremlingo.T__.Select("c").Select("capabilities")). + Property("privileged", gremlingo.T__.Select("c").Select("privileged")). + Property("privesc", gremlingo.T__.Select("c").Select("privesc")). + Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). + Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). + Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). + Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). + // Property(gremlingo.Cardinality.Set, "ports", gremlingo.T__.Select("c").Select("ports")). + Property("pod", gremlingo.T__.Select("c").Select("pod")). + Property("node", gremlingo.T__.Select("c").Select("node")). + Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + Property("critical", gremlingo.T__.Select("c").Select("critical")) - // // The returned promised is a go channel to wait for all submitted steps to finish execution and return error. - // err := <-promise - // if err != nil { - // fmt.Println(err) - // } - // return g.GetGraphTraversal() + return traversal } } diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go index cd823270..fd9aa661 100644 --- a/pkg/kubehound/graph/vertex/container_test.go +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -1,41 +1,28 @@ package vertex import ( + "fmt" "testing" "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" "github.com/DataDog/KubeHound/pkg/utils" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" - "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/assert" ) func TestContainer_Traversal(t *testing.T) { + t.Parallel() + tests := []struct { name string - v Container want VertexTraversal + data graph.Container }{ { name: "Add containers in JanusGraph", - v: Container{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dbHost := "ws://localhost:8182/gremlin" - driver, err := gremlingo.NewDriverRemoteConnection(dbHost) - assert.NoError(t, err) - - g := gremlingo.Traversal_().WithRemote(driver) - // tx := traversal.Tx() - // g, err := tx.Begin() - assert.NoError(t, err) - - v := Container{} - // We set the values to all field with non default values so we are sure all are correctly propagated. - insert, err := utils.StructToMap(graph.Container{ + // We set the values to all field with non default values + // so we are sure all are correctly propagated. + data: graph.Container{ StoreId: "test id", Name: "test name", Image: "image", @@ -54,38 +41,55 @@ func TestContainer_Traversal(t *testing.T) { Node: "test node", Compromised: 1, Critical: true, - }) + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + v := Container{} + + dbHost := "ws://localhost:8182/gremlin" + driver, err := gremlingo.NewDriverRemoteConnection(dbHost) + assert.NoError(t, err) + + g := gremlingo.Traversal_().WithRemote(driver) + assert.NoError(t, err) + + insert, err := utils.StructToMap(tt.data) assert.NoError(t, err) vertexTraversal := v.Traversal() inserts := []TraversalInput{insert} - t.Errorf("inserts: %v", inserts) - _ = vertexTraversal(g, inserts) - // err = tx.Commit() - // assert.NoError(t, err) - // err = tx.Close() - // assert.NoError(t, err) + fmt.Printf("inserts: %v\n", inserts) + traversal := vertexTraversal(g, inserts) - driver, err = gremlingo.NewDriverRemoteConnection(dbHost) + // Write to db + promise := traversal.Iterate() + err = <-promise assert.NoError(t, err) - g = gremlingo.Traversal_().WithRemote(driver) - // tx = traversal.Tx() - // g, err = tx.Begin() + // NO IDEA + // driver, err = gremlingo.NewDriverRemoteConnection(dbHost) + // assert.NoError(t, err) + // g = gremlingo.Traversal_().WithRemote(driver) + // // tx = traversal.Tx() + // // g, err = tx.Begin() - assert.NoError(t, err) - test := g.V().HasLabel(v.Label()).ValueMap() - res, err := test.Traversal.GetResultSet() - assert.NoError(t, err) - data, err := res.All() - for _, d := range data { - thefuck := d.GetInterface() - t.Errorf("thefuck: %+v", thefuck) - container := graph.Container{} - mapstructure.Decode(thefuck, &container) - t.Errorf("container: %+v", container) - } + // assert.NoError(t, err) + // test := g.V().HasLabel(v.Label()).ValueMap() + // res, err := test.Traversal.GetResultSet() + // assert.NoError(t, err) + // data, err := res.All() + // for _, d := range data { + // gotInterface := d.GetInterface().(map[any]any) + // t.Errorf("gotInterface: %+v", gotInterface) + // container := graph.Container{} + // mapstructure.Decode(gotInterface, &container) + // t.Errorf("container: %+v", container) + // } }) } } diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 7c831ebd..c4ed5d4f 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -1,8 +1,8 @@ package vertex import ( - "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) const ( @@ -23,16 +23,15 @@ func (v Node) BatchSize() int { } func (v Node) Traversal() VertexTraversal { - return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - g := source.GetGraphTraversal() - - for _, insert := range inserts { - i := insert.(*graph.Node) - g = g.AddV(v.Label()). - Property("id", i.StoreId). - Property("name", i.Name) - } - - return g + return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + traversal := g.Inject(inserts).Unfold().As("c"). + AddV(v.Label()). + Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + Property("name", gremlingo.T__.Select("c").Select("name")). + Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + Property("namespace", gremlingo.T__.Select("c").Select("namespace")). + Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + Property("critical", gremlingo.T__.Select("c").Select("critical")) + return traversal } } diff --git a/pkg/kubehound/graph/vertex/pod.go b/pkg/kubehound/graph/vertex/pod.go index 4e8d1cda..1de33272 100644 --- a/pkg/kubehound/graph/vertex/pod.go +++ b/pkg/kubehound/graph/vertex/pod.go @@ -1,5 +1,10 @@ package vertex +import ( + gremlin "github.com/apache/tinkerpop/gremlin-go/driver" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) + const ( podLabel = "Pod" ) @@ -18,5 +23,18 @@ func (v Pod) BatchSize() int { } func (v Pod) Traversal() VertexTraversal { - return nil + return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + traversal := g.Inject(inserts).Unfold().As("c"). + AddV(v.Label()). + Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + Property("name", gremlingo.T__.Select("c").Select("name")). + Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + Property("namespace", gremlingo.T__.Select("c").Select("namespace")). + Property("sharedProcessNamespace", gremlingo.T__.Select("c").Select("sharedProcessNamespace")). + Property("serviceAccount", gremlingo.T__.Select("c").Select("serviceAccount")). + Property("node", gremlingo.T__.Select("c").Select("node")). + Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + Property("critical", gremlingo.T__.Select("c").Select("critical")) + return traversal + } } diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index 8663ec25..9ca262e3 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -1,5 +1,10 @@ package vertex +import ( + gremlin "github.com/apache/tinkerpop/gremlin-go/driver" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) + const ( roleLabel = "Role" ) @@ -18,5 +23,14 @@ func (v Role) BatchSize() int { } func (v Role) Traversal() VertexTraversal { - return nil + return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + traversal := g.Inject(inserts).Unfold().As("c"). + AddV(v.Label()). + Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + Property("name", gremlingo.T__.Select("c").Select("name")). + Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + Property("namespace", gremlingo.T__.Select("c").Select("namespace")) + // Property("rules", gremlingo.T__.Select("c").Select("rules")) // array of values + return traversal + } } diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index f73d1381..97c24726 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -1,5 +1,10 @@ package vertex +import ( + gremlin "github.com/apache/tinkerpop/gremlin-go/driver" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) + const ( volumeLabel = "Volume" ) @@ -18,5 +23,13 @@ func (v Volume) BatchSize() int { } func (v Volume) Traversal() VertexTraversal { - return nil + return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + traversal := g.Inject(inserts).Unfold().As("c"). + AddV(v.Label()). + Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + Property("name", gremlingo.T__.Select("c").Select("name")). + Property("type", gremlingo.T__.Select("c").Select("type")). + Property("path", gremlingo.T__.Select("c").Select("path")) + return traversal + } } diff --git a/pkg/utils/interfaces.go b/pkg/utils/interfaces.go new file mode 100644 index 00000000..fc9cfb5c --- /dev/null +++ b/pkg/utils/interfaces.go @@ -0,0 +1,26 @@ +package utils + +import "fmt" + +// func AnySliceToStringSlice(in []any) []string { +// s := make([]string, len(in)) +// for i, v := range in { +// s[i] = fmt.Sprint(v) +// } +// return s +// } +// func AnySliceToIntSlice(in []int64) []int64 { +// s := make([]int, len(in)) +// for i, v := range in { +// s[i] = v +// } +// return s +// } + +func ToSliceOfAny[Tin any](s []Tin) []string { + result := make([]string, len(s)) + for i, v := range s { + result[i] = fmt.Sprint(v) + } + return result +} From 2580f95cc1ce198c1058369d9911134c9349a53d Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 09:54:56 +0200 Subject: [PATCH 15/61] maybe ? --- .../storage/graphdb/janusgraph_provider.go | 4 +- .../storage/graphdb/janusgraph_writer.go | 44 +++++++------------ 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index d9f478b8..5ed5d2cd 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -42,7 +42,7 @@ func (jgp *JanusGraphProvider) Raw() any { } // VertexWriter creates a new AsyncVertexWriter instance to enable asynchronous bulk inserts of vertices. -func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Vertex, opts ...WriterOption) (AsyncVertexWriter, error) { +func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Builder, opts ...WriterOption) (AsyncVertexWriter, error) { writer, err := NewJanusGraphAsyncVertexWriter(jgp.client, opts...) if err != nil { return nil, err @@ -51,7 +51,7 @@ func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Vertex } // EdgeWriter creates a new AsyncEdgeWriter instance to enable asynchronous bulk inserts of edges. -func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Edge, opts ...WriterOption) (AsyncEdgeWriter, error) { +func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Builder, opts ...WriterOption) (AsyncEdgeWriter, error) { writer, err := NewJanusGraphAsyncEdgeWriter(jgp.client) if err != nil { return nil, err diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index fe5a29a4..3ed0b25c 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -11,15 +11,15 @@ import ( var _ AsyncVertexWriter = (*JanusGraphAsyncVertexWriter)(nil) -type GremlinTraversalVertex func(*gremlingo.GraphTraversalSource, []gremlingo.Vertex) *gremlingo.GraphTraversal -type GremlinTraversalEdge func(*gremlingo.GraphTraversalSource, []gremlingo.Edge) *gremlingo.GraphTraversal +type GremlinTraversalVertex func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal +type GremlinTraversalEdge func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal type JanusGraphAsyncVertexWriter struct { gremlin GremlinTraversalVertex transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource - inserts []gremlingo.Vertex // Should this be gremlingo.Edge or vertex.Vertex? - consumerChan chan []gremlingo.Vertex + inserts []any + consumerChan chan []any writingInFligth sync.WaitGroup batchSize int // Shouldn't this be "per vertex types" ? } @@ -30,8 +30,8 @@ type JanusGraphAsyncEdgeWriter struct { gremlin GremlinTraversalEdge transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource - inserts []gremlingo.Edge // Should this be gremlingo.Edge or edge.Edge? - consumerChan chan []gremlingo.Edge + inserts []any + consumerChan chan []any writingInFligth sync.WaitGroup batchSize int // Shouldn't this be "per edge types" ? } @@ -49,7 +49,7 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, opts .. return nil, err } jw := JanusGraphAsyncEdgeWriter{ - inserts: make([]gremlingo.Edge, 0), + inserts: make([]interface{}, 0), transaction: tx, traversalSource: gtx, batchSize: 1, @@ -71,7 +71,7 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts return nil, err } jw := JanusGraphAsyncVertexWriter{ - inserts: make([]gremlingo.Vertex, 0), + inserts: make([]interface{}, 0), transaction: tx, traversalSource: gtx, } @@ -79,7 +79,7 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts return &jw, nil } -func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []gremlingo.Vertex) error { +func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []any) error { jgv.writingInFligth.Add(1) defer jgv.writingInFligth.Done() @@ -94,7 +94,7 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []g return nil } -func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []gremlingo.Edge) error { +func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any) error { jge.writingInFligth.Add(1) defer jge.writingInFligth.Done() @@ -187,19 +187,13 @@ func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { return nil } -func (v *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { - if len(v.inserts) > v.batchSize { - v.consumerChan <- v.inserts +func (vw *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { + if len(vw.inserts) > vw.batchSize { + vw.consumerChan <- vw.inserts // cleanup the ops array after we have copied it to the channel - v.inserts = nil + vw.inserts = nil } - converted := gremlingo.Vertex{ - Element: gremlingo.Element{ - Id: vertex.Label(), //FIXME, not sure what should be added here - Label: vertex.Label(), - }, - } - v.inserts = append(v.inserts, converted) + vw.inserts = append(vw.inserts, vertex) return nil } @@ -209,12 +203,6 @@ func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { // cleanup the ops array after we have copied it to the channel e.inserts = nil } - converted := gremlingo.Edge{ - Element: gremlingo.Element{ - Id: edge.Label(), //FIXME, not sure what should be added here - Label: edge.Label(), - }, - } - e.inserts = append(e.inserts, converted) + e.inserts = append(e.inserts, edge) return nil } From 31fba0f49afe22e5157b4ebc76c5ce437d7d828f Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 10:18:10 +0200 Subject: [PATCH 16/61] remove write to DB --- pkg/kubehound/graph/vertex/container_test.go | 43 ++++---------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go index fd9aa661..ab684ce9 100644 --- a/pkg/kubehound/graph/vertex/container_test.go +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -28,7 +28,7 @@ func TestContainer_Traversal(t *testing.T) { Image: "image", Command: []string{"/usr/bin/sleep"}, Args: []string{"600"}, - Capabilities: []string{"NET_CAP_ADMIN", "NET_RAW_ADMINaa"}, + Capabilities: []string{"NET_CAP_ADMIN", "NET_RAW_ADMIN"}, Privileged: true, PrivEsc: true, HostPID: true, @@ -50,13 +50,7 @@ func TestContainer_Traversal(t *testing.T) { t.Run(tt.name, func(t *testing.T) { v := Container{} - dbHost := "ws://localhost:8182/gremlin" - driver, err := gremlingo.NewDriverRemoteConnection(dbHost) - assert.NoError(t, err) - - g := gremlingo.Traversal_().WithRemote(driver) - assert.NoError(t, err) - + g := gremlingo.GraphTraversalSource{} insert, err := utils.StructToMap(tt.data) assert.NoError(t, err) @@ -64,32 +58,13 @@ func TestContainer_Traversal(t *testing.T) { inserts := []TraversalInput{insert} fmt.Printf("inserts: %v\n", inserts) - traversal := vertexTraversal(g, inserts) - - // Write to db - promise := traversal.Iterate() - err = <-promise - assert.NoError(t, err) - - // NO IDEA - // driver, err = gremlingo.NewDriverRemoteConnection(dbHost) - // assert.NoError(t, err) - // g = gremlingo.Traversal_().WithRemote(driver) - // // tx = traversal.Tx() - // // g, err = tx.Begin() - - // assert.NoError(t, err) - // test := g.V().HasLabel(v.Label()).ValueMap() - // res, err := test.Traversal.GetResultSet() - // assert.NoError(t, err) - // data, err := res.All() - // for _, d := range data { - // gotInterface := d.GetInterface().(map[any]any) - // t.Errorf("gotInterface: %+v", gotInterface) - // container := graph.Container{} - // mapstructure.Decode(gotInterface, &container) - // t.Errorf("container: %+v", container) - // } + traversal := vertexTraversal(&g, inserts) + // This is ugly but doesn't need to write to the DB + // This just makes sure the traversal is correctly returned with the correct values + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "/usr/bin/sleep") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "1337") }) } } From 421f94cdc515584b5ed4cfa765d2695bf3d60cb3 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 10:57:34 +0200 Subject: [PATCH 17/61] Updates --- configs/etc/kubehound.yaml | 2 +- pkg/config/janusgraph.go | 4 ++-- pkg/kubehound/storage/graphdb/janusgraph_provider.go | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/configs/etc/kubehound.yaml b/configs/etc/kubehound.yaml index 00e415a9..8473136e 100644 --- a/configs/etc/kubehound.yaml +++ b/configs/etc/kubehound.yaml @@ -7,7 +7,7 @@ collector: mongodb: url: "mongodb://localhost:27017" janusgraph: - url: "localhost:8182" + url: "ws://localhost:8182/gremlin" telemetry: statsd: url: "127.0.0.1:8125" \ No newline at end of file diff --git a/pkg/config/janusgraph.go b/pkg/config/janusgraph.go index e2105469..577711bc 100644 --- a/pkg/config/janusgraph.go +++ b/pkg/config/janusgraph.go @@ -2,8 +2,8 @@ package config import "time" -// MongoDBConfig configures mongodb specific parameters. +// JanusGraphConfig configures JanusGraph specific parameters. type JanusGraphConfig struct { - URL string `mapstructure:"url"` // Mongodb specific configuration + URL string `mapstructure:"url"` // JanusGraph specific configuration ConnectionTimeout time.Duration `mapstructure:"connection_timeout"` } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 5ed5d2cd..75a31f00 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -2,6 +2,7 @@ package graphdb import ( "context" + "errors" "github.com/DataDog/KubeHound/pkg/globals" "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" @@ -16,6 +17,9 @@ type JanusGraphProvider struct { } func NewGraphDriver(ctx context.Context, dbHost string) (*JanusGraphProvider, error) { + if dbHost == "" { + return nil, errors.New("JanusGraph DB URL is not set") + } driver, err := gremlingo.NewDriverRemoteConnection(dbHost) if err != nil { return nil, err From 23d2b5ca68f5e08db5b986ff0ee9383eb73c40d2 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 11:00:28 +0200 Subject: [PATCH 18/61] Fix config --- test/system/kubehound.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/system/kubehound.yaml b/test/system/kubehound.yaml index cf701f31..8dff7999 100644 --- a/test/system/kubehound.yaml +++ b/test/system/kubehound.yaml @@ -2,6 +2,7 @@ collector: type: file-collector file: directory: kind-collect/ - +janusgraph: + url: "ws://localhost:8182/gremlin" mongodb: url: "mongodb://localhost:27017" \ No newline at end of file From 50238fd0756eaad938bffe291ca40609be8bc4f2 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 11:47:31 +0200 Subject: [PATCH 19/61] updates --- pkg/collector/collector.go | 2 +- pkg/kubehound/storage/graphdb/janusgraph_provider.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 5d4fa83d..d0473b81 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -94,7 +94,7 @@ func ClientFactory(ctx context.Context, cfg *config.KubehoundConfig) (CollectorC switch { case cfg.Collector.Type == config.CollectorTypeK8sAPI: return NewK8sAPICollector(ctx, cfg) - case cfg.Collector.Type == config.CollectorTypeK8sAPI: + case cfg.Collector.Type == config.CollectorTypeFile: return NewFileCollector(ctx, cfg) default: return nil, fmt.Errorf("collector type not supported: %s", cfg.Collector.Type) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 75a31f00..d8b0878e 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/DataDog/KubeHound/pkg/globals" "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" @@ -37,7 +36,10 @@ func (jgp *JanusGraphProvider) Name() string { } func (jgp *JanusGraphProvider) HealthCheck(ctx context.Context) (bool, error) { - return true, globals.ErrNotImplemented + if jgp.client != nil { + return true, nil + } + return false, errors.New("failed to get janus graph client") } // Raw returns a handle to the underlying provider to allow implementation specific operations e.g graph queries. From c9bff2738cb27fcfcac2ac76eb64d0168087101a Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 12:55:24 +0200 Subject: [PATCH 20/61] updates --- pkg/kubehound/storage/graphdb/janusgraph_writer.go | 4 +++- test/system/kubehound.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 3ed0b25c..742c395e 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -49,10 +49,11 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, opts .. return nil, err } jw := JanusGraphAsyncEdgeWriter{ - inserts: make([]interface{}, 0), + inserts: make([]any, 0), transaction: tx, traversalSource: gtx, batchSize: 1, + consumerChan: make(chan []any, 0), } return &jw, nil @@ -74,6 +75,7 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts inserts: make([]interface{}, 0), transaction: tx, traversalSource: gtx, + consumerChan: make(chan []any, 0), } return &jw, nil diff --git a/test/system/kubehound.yaml b/test/system/kubehound.yaml index 8dff7999..793ef172 100644 --- a/test/system/kubehound.yaml +++ b/test/system/kubehound.yaml @@ -1,7 +1,7 @@ collector: type: file-collector file: - directory: kind-collect/ + directory: /home/edouard/dd/KubeHound/test/system/kind-collect/kind-kubehound.test.local janusgraph: url: "ws://localhost:8182/gremlin" mongodb: From 6b7ae92165da3ca88790c2094a5a81fb69fe9876 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 12:57:39 +0200 Subject: [PATCH 21/61] updates --- pkg/kubehound/storage/graphdb/janusgraph_writer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 742c395e..ae414c58 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -75,6 +75,7 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts inserts: make([]interface{}, 0), transaction: tx, traversalSource: gtx, + batchSize: 1, consumerChan: make(chan []any, 0), } From dbc2e250adc86e76eb50a29534a44d664c0f8f30 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 13:03:29 +0200 Subject: [PATCH 22/61] add static buff --- pkg/kubehound/storage/graphdb/janusgraph_writer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index ae414c58..0f4bb366 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -53,7 +53,7 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, opts .. transaction: tx, traversalSource: gtx, batchSize: 1, - consumerChan: make(chan []any, 0), + consumerChan: make(chan []any, 10), } return &jw, nil @@ -76,7 +76,7 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts transaction: tx, traversalSource: gtx, batchSize: 1, - consumerChan: make(chan []any, 0), + consumerChan: make(chan []any, 10), } return &jw, nil From d11442d5d14d7bec0cecf9a4aab848351d4c2229 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Thu, 8 Jun 2023 20:47:34 +0200 Subject: [PATCH 23/61] ???????????????? --- Makefile | 2 +- pkg/kubehound/graph/edge/container_attach.go | 2 + pkg/kubehound/graph/vertex/container.go | 8 ++- pkg/kubehound/graph/vertex/identity.go | 11 +++- pkg/kubehound/graph/vertex/node.go | 5 ++ pkg/kubehound/graph/vertex/pod.go | 5 ++ pkg/kubehound/graph/vertex/role.go | 8 ++- pkg/kubehound/graph/vertex/volume.go | 5 ++ .../ingestor/pipeline/ingest_resources.go | 2 +- pkg/kubehound/ingestor/pipeline/sequence.go | 1 + .../storage/graphdb/janusgraph_provider.go | 4 +- .../storage/graphdb/janusgraph_writer.go | 54 ++++++++++++++----- pkg/utils/interfaces.go | 12 ++++- 13 files changed, 97 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index ac43c3d6..4a09c8ea 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test: .PHONY: system-test system-test: - cd test/system && go test ./... + cd test/system && go test -v -timeout "30s" ./... .PHONY: local-cluster-create local-cluster-setup: diff --git a/pkg/kubehound/graph/edge/container_attach.go b/pkg/kubehound/graph/edge/container_attach.go index c9aa03e8..79fbf973 100644 --- a/pkg/kubehound/graph/edge/container_attach.go +++ b/pkg/kubehound/graph/edge/container_attach.go @@ -5,6 +5,7 @@ import ( "github.com/DataDog/KubeHound/pkg/kubehound/storage/storedb" "github.com/DataDog/KubeHound/pkg/kubehound/store/collections" + "github.com/DataDog/KubeHound/pkg/telemetry/log" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -33,6 +34,7 @@ func (e ContainerAttach) BatchSize() int { func (e ContainerAttach) Traversal() EdgeTraversal { return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + log.I.Errorf("CONVERT ME TO SOMETHING TYPED OTHERWISE THIS WILL BROKE") return g.Inject(inserts).Unfold().As("ca"). V().HasLabel("Pod").Has("storeId", gremlin.T__.Select("ca").Select("pod")).As("pod"). V().HasLabel("Container").Has("storeId", gremlin.T__.Select("ca").Select("container")).As("container"). diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index cfeac48a..bae4de11 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,6 +1,9 @@ package vertex import ( + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + "github.com/DataDog/KubeHound/pkg/telemetry/log" + "github.com/DataDog/KubeHound/pkg/utils" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -61,9 +64,10 @@ func (v Container) Traversal() VertexTraversal { // } // return g.GetGraphTraversal() - // I have no idea how I can convert the arrays in the inserts to multiple values with cardinality set.... - traversal := g.Inject(inserts).Unfold().As("c"). + insertsConverted := utils.ConvertSliceAnyToTyped[graph.Container, TraversalInput](inserts) + log.I.Infof(" ============== INSERTS Containers ====== %+v", insertsConverted) + traversal := g.Inject(insertsConverted).Unfold().As("c"). AddV(v.Label()). Property("storeId", gremlingo.T__.Select("c").Select("store_id")). Property("name", gremlingo.T__.Select("c").Select("name")). diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 5b5ed6b9..f450e230 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -1,6 +1,11 @@ package vertex -import gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +import ( + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + "github.com/DataDog/KubeHound/pkg/telemetry/log" + "github.com/DataDog/KubeHound/pkg/utils" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) const ( identityLabel = "Identity" @@ -21,8 +26,10 @@ func (v Identity) BatchSize() int { func (v Identity) Traversal() VertexTraversal { return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { + insertsConverted := utils.ConvertSliceAnyToTyped[graph.Identity, TraversalInput](inserts) + log.I.Infof(" ============== INSERTS Identities ====== %+v", insertsConverted) return g.Inject(inserts).Unfold().As("c"). - AddV("Identity"). + AddV(v.Label()). Property("storeId", gremlingo.T__.Select("c").Select("store_id")). Property("name", gremlingo.T__.Select("c").Select("name")). Property("isNamespaced", gremlingo.T__.Select("c").Select("is_namespaced")). diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index c4ed5d4f..5fdc3a7b 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -1,6 +1,9 @@ package vertex import ( + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + "github.com/DataDog/KubeHound/pkg/telemetry/log" + "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -24,6 +27,8 @@ func (v Node) BatchSize() int { func (v Node) Traversal() VertexTraversal { return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + insertsConverted := utils.ConvertSliceAnyToTyped[graph.Node, TraversalInput](inserts) + log.I.Infof(" ============== INSERTS Nodes ====== %+v", insertsConverted) traversal := g.Inject(inserts).Unfold().As("c"). AddV(v.Label()). Property("store_id", gremlingo.T__.Select("c").Select("store_id")). diff --git a/pkg/kubehound/graph/vertex/pod.go b/pkg/kubehound/graph/vertex/pod.go index 1de33272..46f6bf5c 100644 --- a/pkg/kubehound/graph/vertex/pod.go +++ b/pkg/kubehound/graph/vertex/pod.go @@ -1,6 +1,9 @@ package vertex import ( + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + "github.com/DataDog/KubeHound/pkg/telemetry/log" + "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -24,6 +27,8 @@ func (v Pod) BatchSize() int { func (v Pod) Traversal() VertexTraversal { return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + insertsConverted := utils.ConvertSliceAnyToTyped[graph.Pod, TraversalInput](inserts) + log.I.Infof(" ============== INSERTS Pods ====== %+v", insertsConverted) traversal := g.Inject(inserts).Unfold().As("c"). AddV(v.Label()). Property("store_id", gremlingo.T__.Select("c").Select("store_id")). diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index 9ca262e3..89824713 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -1,6 +1,9 @@ package vertex import ( + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + "github.com/DataDog/KubeHound/pkg/telemetry/log" + "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -24,7 +27,10 @@ func (v Role) BatchSize() int { func (v Role) Traversal() VertexTraversal { return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - traversal := g.Inject(inserts).Unfold().As("c"). + insertsConverted := utils.ConvertSliceAnyToTyped[graph.Role, TraversalInput](inserts) + log.I.Infof(" ============== INSERTS ROLES ====== %+v", insertsConverted) + + traversal := g.Inject(insertsConverted).Unfold().As("c"). AddV(v.Label()). Property("store_id", gremlingo.T__.Select("c").Select("store_id")). Property("name", gremlingo.T__.Select("c").Select("name")). diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index 97c24726..10a69694 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -1,6 +1,9 @@ package vertex import ( + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + "github.com/DataDog/KubeHound/pkg/telemetry/log" + "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -24,6 +27,8 @@ func (v Volume) BatchSize() int { func (v Volume) Traversal() VertexTraversal { return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + insertsConverted := utils.ConvertSliceAnyToTyped[graph.Volume, TraversalInput](inserts) + log.I.Infof(" ============== INSERTS Volumes ====== %+v", insertsConverted) traversal := g.Inject(inserts).Unfold().As("c"). AddV(v.Label()). Property("store_id", gremlingo.T__.Select("c").Select("store_id")). diff --git a/pkg/kubehound/ingestor/pipeline/ingest_resources.go b/pkg/kubehound/ingestor/pipeline/ingest_resources.go index 1835af50..765f4650 100644 --- a/pkg/kubehound/ingestor/pipeline/ingest_resources.go +++ b/pkg/kubehound/ingestor/pipeline/ingest_resources.go @@ -84,7 +84,7 @@ func WithStoreWriter[T collections.Collection](c T) IngestResourceOption { // WithStoreWriter initializes a bulk graph writer (and registers a cleanup function) for the provided vertex. // To access the writer use the graphWriter(v vertex.Vertex) function. -func WithGraphWriter[T vertex.Builder](v T) IngestResourceOption { +func WithGraphWriter(v vertex.Builder) IngestResourceOption { return func(ctx context.Context, rOpts *resourceOptions, deps *Dependencies) error { w, err := deps.GraphDB.VertexWriter(ctx, v) if err != nil { diff --git a/pkg/kubehound/ingestor/pipeline/sequence.go b/pkg/kubehound/ingestor/pipeline/sequence.go index dc849757..0315810d 100644 --- a/pkg/kubehound/ingestor/pipeline/sequence.go +++ b/pkg/kubehound/ingestor/pipeline/sequence.go @@ -25,6 +25,7 @@ func (s *Sequence) Run(ctx context.Context, deps *Dependencies) error { if err != nil { return fmt.Errorf("group %s ingest: %w", g.Name, err) } + l.Infof("Finished running ingest group %s", g.Name) } l.Infof("Completed ingest sequence %s", s.Name) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index d8b0878e..975bd8fe 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -49,7 +49,7 @@ func (jgp *JanusGraphProvider) Raw() any { // VertexWriter creates a new AsyncVertexWriter instance to enable asynchronous bulk inserts of vertices. func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Builder, opts ...WriterOption) (AsyncVertexWriter, error) { - writer, err := NewJanusGraphAsyncVertexWriter(jgp.client, opts...) + writer, err := NewJanusGraphAsyncVertexWriter(jgp.client, v, opts...) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Builde // EdgeWriter creates a new AsyncEdgeWriter instance to enable asynchronous bulk inserts of edges. func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Builder, opts ...WriterOption) (AsyncEdgeWriter, error) { - writer, err := NewJanusGraphAsyncEdgeWriter(jgp.client) + writer, err := NewJanusGraphAsyncEdgeWriter(jgp.client, e) if err != nil { return nil, err } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 0f4bb366..3ea7af3e 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -5,17 +5,19 @@ import ( "errors" "sync" + "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" + "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" "github.com/DataDog/KubeHound/pkg/telemetry/log" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) var _ AsyncVertexWriter = (*JanusGraphAsyncVertexWriter)(nil) -type GremlinTraversalVertex func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal -type GremlinTraversalEdge func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal +// type GremlinTraversalVertex func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal +// type GremlinTraversalEdge func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal type JanusGraphAsyncVertexWriter struct { - gremlin GremlinTraversalVertex + gremlin vertex.VertexTraversal transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource inserts []any @@ -27,7 +29,7 @@ type JanusGraphAsyncVertexWriter struct { var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) type JanusGraphAsyncEdgeWriter struct { - gremlin GremlinTraversalEdge + gremlin edge.EdgeTraversal transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource inserts []any @@ -36,7 +38,8 @@ type JanusGraphAsyncEdgeWriter struct { batchSize int // Shouldn't this be "per edge types" ? } -func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { +func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, e edge.Builder, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { + log.I.Infof("Created new JanusGraphAsyncEdgeWriter") options := &writerOptions{} for _, opt := range opts { opt(options) @@ -49,6 +52,7 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, opts .. return nil, err } jw := JanusGraphAsyncEdgeWriter{ + gremlin: e.Traversal(), inserts: make([]any, 0), transaction: tx, traversalSource: gtx, @@ -59,19 +63,21 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, opts .. return &jw, nil } -func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { +func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, v vertex.Builder, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { + log.I.Infof("Created new JanusGraphAsyncVertexWriter") options := &writerOptions{} for _, opt := range opts { opt(options) } - traversal := gremlingo.Traversal_().WithRemote(drc) - tx := traversal.Tx() + source := gremlingo.Traversal_().WithRemote(drc) + tx := source.Tx() gtx, err := tx.Begin() if err != nil { return nil, err } jw := JanusGraphAsyncVertexWriter{ + gremlin: v.Traversal(), inserts: make([]interface{}, 0), transaction: tx, traversalSource: gtx, @@ -83,25 +89,48 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, opts } func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []any) error { + log.I.Infof("batch write JanusGraphAsyncVertexWriter") jgv.writingInFligth.Add(1) defer jgv.writingInFligth.Done() - op := jgv.gremlin(jgv.traversalSource, data) + convertedToTraversalInput := make([]vertex.TraversalInput, 0) + for _, d := range data { + convertedToTraversalInput = append(convertedToTraversalInput, d.(vertex.TraversalInput)) + } + + log.I.Infof("BEFORE gremlin()") + op := jgv.gremlin(jgv.traversalSource, convertedToTraversalInput) + log.I.Infof("BEFORE ITERATE") promise := op.Iterate() + log.I.Infof("BEFORE PROMISE") err := <-promise + log.I.Infof("AFTER PROMISE: %v, data: %+v", err, data) if err != nil { jgv.transaction.Rollback() return err } - + log.I.Infof("=== DONE == batch write JanusGraphAsyncVertexWriter") return nil } func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any) error { + log.I.Infof("batch write JanusGraphAsyncEdgeWriter") jge.writingInFligth.Add(1) defer jge.writingInFligth.Done() - op := jge.gremlin(jge.traversalSource, data) + if jge.gremlin == nil { + panic("lol") + } + + // This seems ~pointless BUT is required to have the ability to use edge.TraversalInput/vertex.TraversalInput + // as the type + // Even tho it's an alias to any, since we use it in an array, we cannot simply .([]any) or vice versa because of the underlying memory layout. + convertedToTraversalInput := make([]edge.TraversalInput, 0) + for _, d := range data { + convertedToTraversalInput = append(convertedToTraversalInput, d.(edge.TraversalInput)) + } + + op := jge.gremlin(jge.traversalSource, convertedToTraversalInput) promise := op.Iterate() err := <-promise if err != nil { @@ -109,6 +138,7 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any return err } + log.I.Infof("=== DONE == batch write JanusGraphAsyncEdgeWriter") return nil } @@ -155,7 +185,7 @@ func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { v.writingInFligth.Wait() return nil } - + log.I.Infof("Flushing remaining of queue for vertices: %+v", v.inserts) err := v.batchWrite(ctx, v.inserts) if err != nil { v.writingInFligth.Wait() diff --git a/pkg/utils/interfaces.go b/pkg/utils/interfaces.go index fc9cfb5c..1c078968 100644 --- a/pkg/utils/interfaces.go +++ b/pkg/utils/interfaces.go @@ -1,6 +1,8 @@ package utils -import "fmt" +import ( + "fmt" +) // func AnySliceToStringSlice(in []any) []string { // s := make([]string, len(in)) @@ -24,3 +26,11 @@ func ToSliceOfAny[Tin any](s []Tin) []string { } return result } + +func ConvertSliceAnyToTyped[T any, Tin any](data []Tin) []T { + converted := make([]T, len(data)) + for _, d := range converted { + converted = append(converted, d) + } + return converted +} From 01b5fd6a2eb9026e57808b07757ab8adc89c2bcf Mon Sep 17 00:00:00 2001 From: Jeremy Fox <109584719+d0g0x01@users.noreply.github.com> Date: Fri, 9 Jun 2023 09:05:03 +0100 Subject: [PATCH 24/61] [TEST] attack test infra (#39) --- docs/Architecture.excalidraw | 990 +++++++++--------- test/setup/create-cluster-resources.sh | 10 +- .../test-cluster/attacks/CE_HOST_MOUNT.yaml | 16 + .../test-cluster/attacks/CE_MODULE_LOAD.yaml | 17 + .../test-cluster/attacks/CE_NET_MITM.yaml | 17 + .../test-cluster/attacks/CE_NSENTER.yaml | 17 + .../attacks/CE_UMH_CORE_PATTERN.yaml | 21 + .../attacks/IDENTITY_IMPERSONATE.yaml | 45 + .../test-cluster/attacks/POD_CREATE.yaml | 46 + .../setup/test-cluster/attacks/POD_PATCH.yaml | 45 + .../setup/test-cluster/attacks/ROLE_BIND.yaml | 48 + .../attacks/SHARED_PS_NAMESPACE.yaml | 15 + .../attacks/TOKEN_BRUTEFORCE.yaml | 45 + .../test-cluster/attacks/TOKEN_LIST.yaml | 45 + .../attacks/TOKEN_VAR_LOG_SYMLINK.yaml | 21 + test/setup/test-cluster/hostpath-pod.yaml | 19 - test/setup/test-cluster/priv-pid-pod.yaml | 15 - test/setup/test-cluster/priv-pod.yaml | 14 - 18 files changed, 900 insertions(+), 546 deletions(-) mode change 100644 => 100755 test/setup/create-cluster-resources.sh create mode 100644 test/setup/test-cluster/attacks/CE_HOST_MOUNT.yaml create mode 100644 test/setup/test-cluster/attacks/CE_MODULE_LOAD.yaml create mode 100644 test/setup/test-cluster/attacks/CE_NET_MITM.yaml create mode 100644 test/setup/test-cluster/attacks/CE_NSENTER.yaml create mode 100644 test/setup/test-cluster/attacks/CE_UMH_CORE_PATTERN.yaml create mode 100644 test/setup/test-cluster/attacks/IDENTITY_IMPERSONATE.yaml create mode 100644 test/setup/test-cluster/attacks/POD_CREATE.yaml create mode 100644 test/setup/test-cluster/attacks/POD_PATCH.yaml create mode 100644 test/setup/test-cluster/attacks/ROLE_BIND.yaml create mode 100644 test/setup/test-cluster/attacks/SHARED_PS_NAMESPACE.yaml create mode 100644 test/setup/test-cluster/attacks/TOKEN_BRUTEFORCE.yaml create mode 100644 test/setup/test-cluster/attacks/TOKEN_LIST.yaml create mode 100644 test/setup/test-cluster/attacks/TOKEN_VAR_LOG_SYMLINK.yaml delete mode 100644 test/setup/test-cluster/hostpath-pod.yaml delete mode 100644 test/setup/test-cluster/priv-pid-pod.yaml delete mode 100644 test/setup/test-cluster/priv-pod.yaml diff --git a/docs/Architecture.excalidraw b/docs/Architecture.excalidraw index 3825fd35..f27e7bd4 100644 --- a/docs/Architecture.excalidraw +++ b/docs/Architecture.excalidraw @@ -42,8 +42,8 @@ }, { "type": "text", - "version": 259, - "versionNonce": 1773416908, + "version": 260, + "versionNonce": 1672114560, "isDeleted": false, "id": "_PucrRWt6L-T-qRAMdPqb", "fillStyle": "hachure", @@ -62,7 +62,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340729, + "updated": 1686229367007, "link": null, "locked": false, "fontSize": 28, @@ -118,8 +118,8 @@ }, { "type": "text", - "version": 322, - "versionNonce": 1614389492, + "version": 323, + "versionNonce": 1469015680, "isDeleted": false, "id": "YhDtCGwqFECEp2xAsTldh", "fillStyle": "hachure", @@ -138,7 +138,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340729, + "updated": 1686229367008, "link": null, "locked": false, "fontSize": 28, @@ -1031,8 +1031,8 @@ }, { "type": "text", - "version": 95, - "versionNonce": 308549708, + "version": 96, + "versionNonce": 945832320, "isDeleted": false, "id": "sJO_ZmMepiyi06AHrKkQ9", "fillStyle": "hachure", @@ -1060,7 +1060,7 @@ "type": "arrow" } ], - "updated": 1684138340729, + "updated": 1686229367008, "link": null, "locked": false, "fontSize": 28, @@ -1175,8 +1175,8 @@ }, { "type": "text", - "version": 94, - "versionNonce": 1203855988, + "version": 95, + "versionNonce": 769562240, "isDeleted": false, "id": "gBl7iXodnSZqHQ1-prZKs", "fillStyle": "hachure", @@ -1195,7 +1195,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340730, + "updated": 1686229367008, "link": null, "locked": false, "fontSize": 28, @@ -1210,8 +1210,8 @@ }, { "type": "text", - "version": 165, - "versionNonce": 424881868, + "version": 166, + "versionNonce": 366743936, "isDeleted": false, "id": "MhOA3DvDyl9_PMA_eGylg", "fillStyle": "hachure", @@ -1230,7 +1230,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340730, + "updated": 1686229367008, "link": null, "locked": false, "fontSize": 28, @@ -1297,8 +1297,8 @@ }, { "type": "text", - "version": 44, - "versionNonce": 1003811828, + "version": 45, + "versionNonce": 1388824192, "isDeleted": false, "id": "rbXLnscTYTQJbUSpqPlRA", "fillStyle": "hachure", @@ -1317,7 +1317,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340730, + "updated": 1686229367009, "link": null, "locked": false, "fontSize": 28, @@ -1383,8 +1383,8 @@ }, { "type": "text", - "version": 80, - "versionNonce": 1327199564, + "version": 81, + "versionNonce": 1935532416, "isDeleted": false, "id": "GTnv7qVfRupMg-qr0ON_9", "fillStyle": "hachure", @@ -1403,7 +1403,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340730, + "updated": 1686229367009, "link": null, "locked": false, "fontSize": 28, @@ -1474,8 +1474,8 @@ }, { "type": "text", - "version": 34, - "versionNonce": 1517966708, + "version": 35, + "versionNonce": 1661662848, "isDeleted": false, "id": "3fJTm7yFDDPBsPDnKisNn", "fillStyle": "hachure", @@ -1494,7 +1494,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340730, + "updated": 1686229367009, "link": null, "locked": false, "fontSize": 28, @@ -1612,8 +1612,8 @@ }, { "type": "text", - "version": 33, - "versionNonce": 1628892108, + "version": 34, + "versionNonce": 1613083008, "isDeleted": false, "id": "5E4xFMUwwNQLY2aXfxg-F", "fillStyle": "hachure", @@ -1632,7 +1632,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340730, + "updated": 1686229367009, "link": null, "locked": false, "fontSize": 28, @@ -1699,8 +1699,8 @@ }, { "type": "text", - "version": 55, - "versionNonce": 284657396, + "version": 56, + "versionNonce": 1667769984, "isDeleted": false, "id": "vnE1EeMRT6AWoC-scGFOM", "fillStyle": "hachure", @@ -1719,7 +1719,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367009, "link": null, "locked": false, "fontSize": 28, @@ -1786,8 +1786,8 @@ }, { "type": "text", - "version": 56, - "versionNonce": 1096782412, + "version": 57, + "versionNonce": 57901440, "isDeleted": false, "id": "grEzh-uvqL6wFzulPwS0q", "fillStyle": "hachure", @@ -1806,7 +1806,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367009, "link": null, "locked": false, "fontSize": 28, @@ -1862,8 +1862,8 @@ }, { "type": "text", - "version": 276, - "versionNonce": 869435508, + "version": 277, + "versionNonce": 1084184192, "isDeleted": false, "id": "9V4NdS3ncIeKw-ct3LuKS", "fillStyle": "hachure", @@ -1882,7 +1882,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -1976,8 +1976,8 @@ }, { "type": "text", - "version": 109, - "versionNonce": 248473804, + "version": 110, + "versionNonce": 478630272, "isDeleted": false, "id": "uJatiRDv4sNvMEWLmFCZs", "fillStyle": "hachure", @@ -1996,7 +1996,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2133,8 +2133,8 @@ }, { "type": "text", - "version": 97, - "versionNonce": 382818804, + "version": 98, + "versionNonce": 1332774528, "isDeleted": false, "id": "qsY95SxOH_-1MsBxffwR8", "fillStyle": "hachure", @@ -2153,7 +2153,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2313,8 +2313,8 @@ }, { "type": "text", - "version": 304, - "versionNonce": 2097180492, + "version": 305, + "versionNonce": 44114304, "isDeleted": false, "id": "xj-u_gjSnmn3GDxyNR2d8", "fillStyle": "hachure", @@ -2333,7 +2333,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2348,8 +2348,8 @@ }, { "type": "text", - "version": 277, - "versionNonce": 620362612, + "version": 278, + "versionNonce": 1285273216, "isDeleted": false, "id": "5RMFef4-FX4TQRA_fiGXq", "fillStyle": "hachure", @@ -2368,7 +2368,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2444,8 +2444,8 @@ }, { "type": "text", - "version": 294, - "versionNonce": 1011656140, + "version": 295, + "versionNonce": 620818816, "isDeleted": false, "id": "Q8HFSir8MT7mNfc1xSlP0", "fillStyle": "hachure", @@ -2464,7 +2464,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340731, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2540,8 +2540,8 @@ }, { "type": "text", - "version": 308, - "versionNonce": 2022862068, + "version": 309, + "versionNonce": 1558242944, "isDeleted": false, "id": "Xx8EMOQ-0nia17TSOBruD", "fillStyle": "hachure", @@ -2560,7 +2560,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2636,8 +2636,8 @@ }, { "type": "text", - "version": 328, - "versionNonce": 1640893516, + "version": 329, + "versionNonce": 182894976, "isDeleted": false, "id": "OYk4eZYTSZz7Z8N1q51Im", "fillStyle": "hachure", @@ -2656,7 +2656,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2732,8 +2732,8 @@ }, { "type": "text", - "version": 315, - "versionNonce": 1931521652, + "version": 316, + "versionNonce": 218109568, "isDeleted": false, "id": "THQTcyKGefUDkWgebQCI_", "fillStyle": "hachure", @@ -2752,7 +2752,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367010, "link": null, "locked": false, "fontSize": 28, @@ -2828,8 +2828,8 @@ }, { "type": "text", - "version": 307, - "versionNonce": 1331602124, + "version": 308, + "versionNonce": 1402114432, "isDeleted": false, "id": "eRAlYYwrSvaomNqigGNo_", "fillStyle": "hachure", @@ -2848,7 +2848,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -2924,8 +2924,8 @@ }, { "type": "text", - "version": 328, - "versionNonce": 1433059316, + "version": 329, + "versionNonce": 1371096704, "isDeleted": false, "id": "ae6RAopSKrGrhlpEiBhp9", "fillStyle": "hachure", @@ -2944,7 +2944,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3065,8 +3065,8 @@ }, { "type": "text", - "version": 379, - "versionNonce": 462973260, + "version": 380, + "versionNonce": 910451072, "isDeleted": false, "id": "2Qvkr2NhtfGOcw3dpgKc-", "fillStyle": "hachure", @@ -3085,7 +3085,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3100,8 +3100,8 @@ }, { "type": "text", - "version": 345, - "versionNonce": 1467392372, + "version": 346, + "versionNonce": 130840192, "isDeleted": false, "id": "uIC8FEvSKMfK2mGhnRdDZ", "fillStyle": "hachure", @@ -3120,7 +3120,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3196,8 +3196,8 @@ }, { "type": "text", - "version": 380, - "versionNonce": 2044026828, + "version": 381, + "versionNonce": 37499264, "isDeleted": false, "id": "X1YLHU7ohQU4sOcnHA0qv", "fillStyle": "hachure", @@ -3216,7 +3216,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3292,8 +3292,8 @@ }, { "type": "text", - "version": 398, - "versionNonce": 1082807028, + "version": 399, + "versionNonce": 1945740928, "isDeleted": false, "id": "jeCM6c0i4Y4GPlsfWd4VP", "fillStyle": "hachure", @@ -3312,7 +3312,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340732, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3388,8 +3388,8 @@ }, { "type": "text", - "version": 382, - "versionNonce": 573539916, + "version": 383, + "versionNonce": 515506560, "isDeleted": false, "id": "vy6xQkiQNebtwi6MR1KA6", "fillStyle": "hachure", @@ -3413,7 +3413,7 @@ "type": "arrow" } ], - "updated": 1684138340732, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3489,8 +3489,8 @@ }, { "type": "text", - "version": 379, - "versionNonce": 2133578868, + "version": 380, + "versionNonce": 1124128384, "isDeleted": false, "id": "9o8Nx_3Aqci2cLWRVw73U", "fillStyle": "hachure", @@ -3509,7 +3509,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3585,8 +3585,8 @@ }, { "type": "text", - "version": 398, - "versionNonce": 964746444, + "version": 399, + "versionNonce": 184406400, "isDeleted": false, "id": "62qOOnavk9Qc0c1Qulbt9", "fillStyle": "hachure", @@ -3605,7 +3605,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -3681,8 +3681,8 @@ }, { "type": "text", - "version": 394, - "versionNonce": 1092691444, + "version": 395, + "versionNonce": 161613440, "isDeleted": false, "id": "J24wke_9NjpUX88qpjt2a", "fillStyle": "hachure", @@ -3701,7 +3701,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367011, "link": null, "locked": false, "fontSize": 28, @@ -4671,8 +4671,8 @@ }, { "type": "text", - "version": 85, - "versionNonce": 962814796, + "version": 86, + "versionNonce": 1581752704, "isDeleted": false, "id": "HapGfrQFrDl5KE-Oby23Z", "fillStyle": "hachure", @@ -4696,7 +4696,7 @@ "type": "arrow" } ], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -4711,8 +4711,8 @@ }, { "type": "text", - "version": 130, - "versionNonce": 191907700, + "version": 131, + "versionNonce": 1151153792, "isDeleted": false, "id": "Yq7rXU67DrJQRiPk7DO4o", "fillStyle": "hachure", @@ -4731,7 +4731,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -4789,8 +4789,8 @@ }, { "type": "text", - "version": 80, - "versionNonce": 1479612876, + "version": 81, + "versionNonce": 1205301632, "isDeleted": false, "id": "cLd1O2Lwxzt4GEBAr_K4R", "fillStyle": "hachure", @@ -4809,7 +4809,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -4867,8 +4867,8 @@ }, { "type": "text", - "version": 133, - "versionNonce": 1381972212, + "version": 134, + "versionNonce": 1488086656, "isDeleted": false, "id": "7W32qHrGktfKhsDLb5QvA", "fillStyle": "hachure", @@ -4887,7 +4887,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -4945,8 +4945,8 @@ }, { "type": "text", - "version": 131, - "versionNonce": 1896087628, + "version": 132, + "versionNonce": 250429824, "isDeleted": false, "id": "dni7LEI81rGkEtw9q3eZN", "fillStyle": "hachure", @@ -4965,7 +4965,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -5023,8 +5023,8 @@ }, { "type": "text", - "version": 125, - "versionNonce": 292813428, + "version": 126, + "versionNonce": 312579712, "isDeleted": false, "id": "rvw_FkiMnZ6xfJUCssEnW", "fillStyle": "hachure", @@ -5043,7 +5043,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -5114,8 +5114,8 @@ }, { "type": "text", - "version": 94, - "versionNonce": 1807095500, + "version": 95, + "versionNonce": 315167104, "isDeleted": false, "id": "v63LJ3Vzev4f3cKRNbejM", "fillStyle": "hachure", @@ -5134,7 +5134,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -5177,8 +5177,8 @@ }, { "type": "text", - "version": 104, - "versionNonce": 1668582388, + "version": 105, + "versionNonce": 657114752, "isDeleted": false, "id": "YeBADWHp2vZRmKb-TD44J", "fillStyle": "hachure", @@ -5197,7 +5197,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340733, + "updated": 1686229367012, "link": null, "locked": false, "fontSize": 28, @@ -5296,8 +5296,8 @@ }, { "type": "text", - "version": 439, - "versionNonce": 618749260, + "version": 440, + "versionNonce": 1252712832, "isDeleted": false, "id": "0nXep7d3EFLCFaHLDUSx_", "fillStyle": "hachure", @@ -5316,7 +5316,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340734, + "updated": 1686229367013, "link": null, "locked": false, "fontSize": 20, @@ -7002,8 +7002,8 @@ }, { "type": "text", - "version": 133, - "versionNonce": 442953076, + "version": 134, + "versionNonce": 856553088, "isDeleted": false, "id": "6Su8vpgYUqsmtknzq5VDS", "fillStyle": "hachure", @@ -7022,7 +7022,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340734, + "updated": 1686229367013, "link": null, "locked": false, "fontSize": 20, @@ -8708,8 +8708,8 @@ }, { "type": "text", - "version": 188, - "versionNonce": 1440732108, + "version": 189, + "versionNonce": 1171435904, "isDeleted": false, "id": "zZPs1bg4kgerfBJZlmBts", "fillStyle": "hachure", @@ -8728,7 +8728,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340734, + "updated": 1686229367013, "link": null, "locked": false, "fontSize": 20, @@ -10414,8 +10414,8 @@ }, { "type": "text", - "version": 296, - "versionNonce": 1469128436, + "version": 297, + "versionNonce": 990586496, "isDeleted": false, "id": "h749tTzy4MnTdUx-ebPlM", "fillStyle": "hachure", @@ -10434,7 +10434,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340735, + "updated": 1686229367013, "link": null, "locked": false, "fontSize": 20, @@ -12120,8 +12120,8 @@ }, { "type": "text", - "version": 310, - "versionNonce": 2093709900, + "version": 311, + "versionNonce": 729842048, "isDeleted": false, "id": "_IkzPk-IXZOv1AkhgTZlJ", "fillStyle": "hachure", @@ -12140,7 +12140,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340735, + "updated": 1686229367013, "link": null, "locked": false, "fontSize": 20, @@ -12192,8 +12192,8 @@ }, { "type": "text", - "version": 104, - "versionNonce": 595140724, + "version": 105, + "versionNonce": 736253568, "isDeleted": false, "id": "fhtnQZ7xkE4vZzHVHaQXd", "fillStyle": "hachure", @@ -12212,7 +12212,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340735, + "updated": 1686229367013, "link": null, "locked": false, "fontSize": 20, @@ -12227,8 +12227,8 @@ }, { "type": "text", - "version": 176, - "versionNonce": 1149128908, + "version": 177, + "versionNonce": 989090176, "isDeleted": false, "id": "_awvnh49jf-mf_Dz4IlKh", "fillStyle": "hachure", @@ -12247,7 +12247,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340735, + "updated": 1686229367013, "link": null, "locked": false, "fontSize": 20, @@ -12678,8 +12678,8 @@ }, { "type": "text", - "version": 34, - "versionNonce": 98890228, + "version": 35, + "versionNonce": 1515423360, "isDeleted": false, "id": "-GHqS8t7mzHIfqTsZnevk", "fillStyle": "hachure", @@ -12703,7 +12703,7 @@ "type": "arrow" } ], - "updated": 1684138340735, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -13106,8 +13106,8 @@ }, { "type": "text", - "version": 106, - "versionNonce": 1821529932, + "version": 107, + "versionNonce": 1265508736, "isDeleted": false, "id": "CvDdQVlqAFXOuaA4tKvyT", "fillStyle": "hachure", @@ -13126,7 +13126,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340735, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -13141,8 +13141,8 @@ }, { "type": "text", - "version": 125, - "versionNonce": 500863860, + "version": 126, + "versionNonce": 52344448, "isDeleted": false, "id": "XbeLWyVem7v6CSSyR3WKn", "fillStyle": "hachure", @@ -13161,7 +13161,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340735, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -13221,8 +13221,8 @@ }, { "type": "text", - "version": 68, - "versionNonce": 828476876, + "version": 69, + "versionNonce": 1278079360, "isDeleted": false, "id": "EO8bRIiEmk7DWH7DzMvej", "fillStyle": "hachure", @@ -13241,7 +13241,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340735, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -13851,8 +13851,8 @@ }, { "type": "text", - "version": 79, - "versionNonce": 966788340, + "version": 80, + "versionNonce": 1258546816, "isDeleted": false, "id": "uAQf51mOXJz4uyUjWoj55", "fillStyle": "hachure", @@ -13871,7 +13871,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -13914,8 +13914,8 @@ }, { "type": "text", - "version": 36, - "versionNonce": 872805452, + "version": 37, + "versionNonce": 1148436864, "isDeleted": false, "id": "zEuehdc3bMVftmIh2a_Lj", "fillStyle": "hachure", @@ -13934,7 +13934,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -13977,8 +13977,8 @@ }, { "type": "text", - "version": 20, - "versionNonce": 1857767028, + "version": 21, + "versionNonce": 1052972672, "isDeleted": false, "id": "-NowFvWoyGBcG4bE3Eoe3", "fillStyle": "hachure", @@ -13997,7 +13997,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -14045,8 +14045,8 @@ }, { "type": "text", - "version": 86, - "versionNonce": 1650035404, + "version": 87, + "versionNonce": 1400869248, "isDeleted": false, "id": "KZiBuznSWGBjrOkeluaJh", "fillStyle": "hachure", @@ -14065,7 +14065,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -14113,8 +14113,8 @@ }, { "type": "text", - "version": 80, - "versionNonce": 837965812, + "version": 81, + "versionNonce": 1394361984, "isDeleted": false, "id": "Cz3o8zCBmAKr3xDGJhqB2", "fillStyle": "hachure", @@ -14133,7 +14133,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -15769,8 +15769,8 @@ }, { "type": "text", - "version": 61, - "versionNonce": 1593200972, + "version": 62, + "versionNonce": 814834048, "isDeleted": false, "id": "TUqxZ7SnpcRy9MvMax_BO", "fillStyle": "hachure", @@ -15789,7 +15789,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367014, "link": null, "locked": false, "fontSize": 20, @@ -15804,8 +15804,8 @@ }, { "type": "text", - "version": 95, - "versionNonce": 1867200884, + "version": 96, + "versionNonce": 1691317888, "isDeleted": false, "id": "6xdRAgKtKnk3_eGkmo0jO", "fillStyle": "hachure", @@ -15824,7 +15824,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367015, "link": null, "locked": false, "fontSize": 20, @@ -15880,8 +15880,8 @@ }, { "type": "text", - "version": 380, - "versionNonce": 1765165004, + "version": 381, + "versionNonce": 719925632, "isDeleted": false, "id": "egwDDncYOG_tQV-kAu_Jq", "fillStyle": "hachure", @@ -15900,7 +15900,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367015, "link": null, "locked": false, "fontSize": 28, @@ -16383,8 +16383,8 @@ }, { "type": "text", - "version": 248, - "versionNonce": 571206388, + "version": 249, + "versionNonce": 949790336, "isDeleted": false, "id": "Howa8TQy9WlNeQY7IFhdI", "fillStyle": "hachure", @@ -16403,7 +16403,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367015, "link": null, "locked": false, "fontSize": 28, @@ -16418,8 +16418,8 @@ }, { "type": "text", - "version": 186, - "versionNonce": 1160998476, + "version": 187, + "versionNonce": 700907904, "isDeleted": false, "id": "-kMNckUw3lQqSqTjAISxd", "fillStyle": "hachure", @@ -16438,7 +16438,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340736, + "updated": 1686229367039, "link": null, "locked": false, "fontSize": 28, @@ -16547,8 +16547,8 @@ }, { "type": "text", - "version": 219, - "versionNonce": 1824156788, + "version": 220, + "versionNonce": 2068043392, "isDeleted": false, "id": "YPYEjVuQFyVe2PI-WkEpo", "fillStyle": "hachure", @@ -16572,7 +16572,7 @@ "type": "arrow" } ], - "updated": 1684138340737, + "updated": 1686229367040, "link": null, "locked": false, "fontSize": 28, @@ -16634,8 +16634,8 @@ }, { "type": "text", - "version": 146, - "versionNonce": 600294604, + "version": 147, + "versionNonce": 745197952, "isDeleted": false, "id": "xBqw9q_jDO4kcgiPZ-6nK", "fillStyle": "hachure", @@ -16663,7 +16663,7 @@ "type": "arrow" } ], - "updated": 1684138340737, + "updated": 1686229367054, "link": null, "locked": false, "fontSize": 28, @@ -16729,8 +16729,8 @@ }, { "type": "text", - "version": 150, - "versionNonce": 823967220, + "version": 151, + "versionNonce": 1099236992, "isDeleted": false, "id": "aC1hBzszkK5enBuqWlkCK", "fillStyle": "hachure", @@ -16758,7 +16758,7 @@ "type": "arrow" } ], - "updated": 1684138340737, + "updated": 1686229367054, "link": null, "locked": false, "fontSize": 28, @@ -16820,8 +16820,8 @@ }, { "type": "text", - "version": 109, - "versionNonce": 177032012, + "version": 110, + "versionNonce": 1242866048, "isDeleted": false, "id": "2SRBMgGBS9HO5oBhmR1Pm", "fillStyle": "hachure", @@ -16849,7 +16849,7 @@ "type": "arrow" } ], - "updated": 1684138340737, + "updated": 1686229367055, "link": null, "locked": false, "fontSize": 28, @@ -17250,8 +17250,8 @@ }, { "type": "text", - "version": 171, - "versionNonce": 674815860, + "version": 172, + "versionNonce": 136328832, "isDeleted": false, "id": "AjyE3hskerq5S0alAsJLS", "fillStyle": "hachure", @@ -17270,7 +17270,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340737, + "updated": 1686229367055, "link": null, "locked": false, "fontSize": 28, @@ -17383,8 +17383,8 @@ }, { "type": "text", - "version": 193, - "versionNonce": 1004405196, + "version": 194, + "versionNonce": 839152000, "isDeleted": false, "id": "2ImcFmDw5TePp9f6hhi4A", "fillStyle": "hachure", @@ -17416,7 +17416,7 @@ "type": "arrow" } ], - "updated": 1684138340737, + "updated": 1686229367055, "link": null, "locked": false, "fontSize": 28, @@ -17822,8 +17822,8 @@ }, { "type": "text", - "version": 331, - "versionNonce": 2052378868, + "version": 332, + "versionNonce": 869623424, "isDeleted": false, "id": "ri5yBasROMNLv3BqfTkFY", "fillStyle": "hachure", @@ -17842,7 +17842,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340737, + "updated": 1686229367058, "link": null, "locked": false, "fontSize": 28, @@ -17904,8 +17904,8 @@ }, { "type": "text", - "version": 70, - "versionNonce": 517204044, + "version": 71, + "versionNonce": 729432448, "isDeleted": false, "id": "ZIXo5j24FVchmuML-JqrO", "fillStyle": "hachure", @@ -17924,7 +17924,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340737, + "updated": 1686229367059, "link": null, "locked": false, "fontSize": 28, @@ -19541,8 +19541,8 @@ }, { "type": "text", - "version": 409, - "versionNonce": 1458999924, + "version": 410, + "versionNonce": 291804800, "isDeleted": false, "id": "XTD_gXU_gSq9V87gFisld", "fillStyle": "hachure", @@ -19561,7 +19561,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340737, + "updated": 1686229367064, "link": null, "locked": false, "fontSize": 28, @@ -19665,8 +19665,8 @@ }, { "type": "text", - "version": 428, - "versionNonce": 659095244, + "version": 429, + "versionNonce": 364253568, "isDeleted": false, "id": "z34KgsXM2TscgIAB9ooPf", "fillStyle": "hachure", @@ -19685,7 +19685,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340737, + "updated": 1686229367064, "link": null, "locked": false, "fontSize": 28, @@ -19756,8 +19756,8 @@ }, { "type": "text", - "version": 213, - "versionNonce": 216278004, + "version": 214, + "versionNonce": 1435354752, "isDeleted": false, "id": "B8jxWbNMvMadzhP7uXDvD", "fillStyle": "hachure", @@ -19776,7 +19776,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367064, "link": null, "locked": false, "fontSize": 28, @@ -19791,8 +19791,8 @@ }, { "type": "text", - "version": 280, - "versionNonce": 1037518156, + "version": 281, + "versionNonce": 1744298368, "isDeleted": false, "id": "6vzmClW-Ro25lbW1APTCN", "fillStyle": "hachure", @@ -19811,7 +19811,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367064, "link": null, "locked": false, "fontSize": 28, @@ -20142,8 +20142,8 @@ }, { "type": "text", - "version": 337, - "versionNonce": 572753268, + "version": 338, + "versionNonce": 487650944, "isDeleted": false, "id": "N9DAmwkBPCqomDAqz73Da", "fillStyle": "hachure", @@ -20162,7 +20162,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367064, "link": null, "locked": false, "fontSize": 28, @@ -20177,8 +20177,8 @@ }, { "type": "text", - "version": 260, - "versionNonce": 668515276, + "version": 261, + "versionNonce": 830452096, "isDeleted": false, "id": "nac6VXoUQeADi7V2tIAyL", "fillStyle": "hachure", @@ -20197,7 +20197,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -20377,8 +20377,8 @@ }, { "type": "text", - "version": 208, - "versionNonce": 1811592948, + "version": 209, + "versionNonce": 1823352448, "isDeleted": false, "id": "9IjYGkrINpvrozMn11yMt", "fillStyle": "hachure", @@ -20397,7 +20397,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -20459,8 +20459,8 @@ }, { "type": "text", - "version": 203, - "versionNonce": 1869046348, + "version": 204, + "versionNonce": 428704128, "isDeleted": false, "id": "x4RtzbNrw0yT8GxZc_4P0", "fillStyle": "hachure", @@ -20479,7 +20479,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -20550,8 +20550,8 @@ }, { "type": "text", - "version": 283, - "versionNonce": 653244532, + "version": 284, + "versionNonce": 824530560, "isDeleted": false, "id": "InMAnX4wYDihesboentS9", "fillStyle": "hachure", @@ -20570,7 +20570,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -20901,8 +20901,8 @@ }, { "type": "text", - "version": 376, - "versionNonce": 1264400588, + "version": 377, + "versionNonce": 1600213376, "isDeleted": false, "id": "xwwPm3GU_sRzaHk1Dnh99", "fillStyle": "hachure", @@ -20926,7 +20926,7 @@ "type": "arrow" } ], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -20941,8 +20941,8 @@ }, { "type": "text", - "version": 332, - "versionNonce": 248023540, + "version": 333, + "versionNonce": 1060537984, "isDeleted": false, "id": "F5igZKg0oi_RRfvl45Vf8", "fillStyle": "hachure", @@ -20961,7 +20961,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -21028,8 +21028,8 @@ }, { "type": "text", - "version": 287, - "versionNonce": 122961740, + "version": 288, + "versionNonce": 1513824640, "isDeleted": false, "id": "bea4jMjfTNStdMK4C0BY9", "fillStyle": "hachure", @@ -21048,7 +21048,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -21106,8 +21106,8 @@ }, { "type": "text", - "version": 306, - "versionNonce": 1988832116, + "version": 307, + "versionNonce": 1403106944, "isDeleted": false, "id": "Okv5hntUC3DyIfWDsFiqX", "fillStyle": "hachure", @@ -21126,7 +21126,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340738, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -21197,8 +21197,8 @@ }, { "type": "text", - "version": 356, - "versionNonce": 1806071244, + "version": 357, + "versionNonce": 2036003200, "isDeleted": false, "id": "UN-lpZ0wRvxeeIpI7thIK", "fillStyle": "hachure", @@ -21217,7 +21217,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367065, "link": null, "locked": false, "fontSize": 28, @@ -21552,8 +21552,8 @@ }, { "type": "text", - "version": 543, - "versionNonce": 1618844916, + "version": 544, + "versionNonce": 321316480, "isDeleted": false, "id": "E1sVYxk44xwhXS_sP4oVn", "fillStyle": "hachure", @@ -21572,7 +21572,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -21635,8 +21635,8 @@ }, { "type": "text", - "version": 497, - "versionNonce": 627956812, + "version": 498, + "versionNonce": 1140900224, "isDeleted": false, "id": "47XSK4_kp_vso-uwDQ627", "fillStyle": "hachure", @@ -21655,7 +21655,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -21717,8 +21717,8 @@ }, { "type": "text", - "version": 513, - "versionNonce": 371580532, + "version": 514, + "versionNonce": 176559744, "isDeleted": false, "id": "JGiYtNmfw8M5Qu-vkVmHh", "fillStyle": "hachure", @@ -21737,7 +21737,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -22200,8 +22200,8 @@ }, { "type": "text", - "version": 566, - "versionNonce": 780432076, + "version": 567, + "versionNonce": 1500287360, "isDeleted": false, "id": "9t203Yx2ASc2Fnt7gcIs8", "fillStyle": "hachure", @@ -22220,7 +22220,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -22603,8 +22603,8 @@ }, { "type": "text", - "version": 503, - "versionNonce": 1078587380, + "version": 504, + "versionNonce": 780093056, "isDeleted": false, "id": "hGRMhsF2Lt6tSi54-rnM5", "fillStyle": "hachure", @@ -22623,7 +22623,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -22638,8 +22638,8 @@ }, { "type": "text", - "version": 520, - "versionNonce": 897857868, + "version": 521, + "versionNonce": 1893622144, "isDeleted": false, "id": "Z87U81mHKsNQZGda8v-5z", "fillStyle": "hachure", @@ -22658,7 +22658,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -22720,8 +22720,8 @@ }, { "type": "text", - "version": 302, - "versionNonce": 2129645940, + "version": 303, + "versionNonce": 1540519552, "isDeleted": false, "id": "75MkHAs2PaKs_YExiQaNq", "fillStyle": "hachure", @@ -22740,7 +22740,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -22755,8 +22755,8 @@ }, { "type": "text", - "version": 335, - "versionNonce": 96939980, + "version": 336, + "versionNonce": 1503015296, "isDeleted": false, "id": "4pxs1fhCbqEa0TB45EMLP", "fillStyle": "hachure", @@ -22775,7 +22775,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367066, "link": null, "locked": false, "fontSize": 28, @@ -23069,8 +23069,8 @@ }, { "type": "text", - "version": 358, - "versionNonce": 22905588, + "version": 359, + "versionNonce": 1463789184, "isDeleted": false, "id": "eXXqVLxMZ-SINaZWEK6vz", "fillStyle": "hachure", @@ -23089,7 +23089,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367071, "link": null, "locked": false, "fontSize": 28, @@ -23383,8 +23383,8 @@ }, { "type": "text", - "version": 383, - "versionNonce": 1869043276, + "version": 384, + "versionNonce": 2060714368, "isDeleted": false, "id": "eJdETe_wMa0w2MxgbDDoe", "fillStyle": "hachure", @@ -23403,7 +23403,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340739, + "updated": 1686229367071, "link": null, "locked": false, "fontSize": 28, @@ -23697,8 +23697,8 @@ }, { "type": "text", - "version": 441, - "versionNonce": 504956020, + "version": 442, + "versionNonce": 1300682368, "isDeleted": false, "id": "l_FDOh7PwBZ_RnXBuq1rf", "fillStyle": "hachure", @@ -23717,7 +23717,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367071, "link": null, "locked": false, "fontSize": 28, @@ -24011,8 +24011,8 @@ }, { "type": "text", - "version": 479, - "versionNonce": 792636620, + "version": 480, + "versionNonce": 1406652800, "isDeleted": false, "id": "jU6YsQJXfXX8LMOST6zpc", "fillStyle": "hachure", @@ -24031,7 +24031,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367072, "link": null, "locked": false, "fontSize": 28, @@ -24325,8 +24325,8 @@ }, { "type": "text", - "version": 537, - "versionNonce": 1793611252, + "version": 538, + "versionNonce": 1399326336, "isDeleted": false, "id": "JofDC9OYeEb-ry7vn6txt", "fillStyle": "hachure", @@ -24345,7 +24345,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367072, "link": null, "locked": false, "fontSize": 28, @@ -24672,8 +24672,8 @@ }, { "type": "text", - "version": 663, - "versionNonce": 1457992524, + "version": 664, + "versionNonce": 2078384512, "isDeleted": false, "id": "NkUfhIQFMwSI6LA0f2XAm", "fillStyle": "hachure", @@ -24692,7 +24692,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367072, "link": null, "locked": false, "fontSize": 28, @@ -25084,8 +25084,8 @@ }, { "type": "text", - "version": 767, - "versionNonce": 1423013748, + "version": 768, + "versionNonce": 1705195136, "isDeleted": false, "id": "1meiXjy5JzkE6iEHTfWDR", "fillStyle": "hachure", @@ -25104,7 +25104,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367072, "link": null, "locked": false, "fontSize": 28, @@ -25241,8 +25241,8 @@ }, { "type": "text", - "version": 324, - "versionNonce": 884664780, + "version": 325, + "versionNonce": 573665664, "isDeleted": false, "id": "DgCGpzLU9rBvN5xs2dtna", "fillStyle": "hachure", @@ -25261,7 +25261,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367072, "link": null, "locked": false, "fontSize": 28, @@ -25791,8 +25791,8 @@ }, { "type": "text", - "version": 372, - "versionNonce": 941254900, + "version": 373, + "versionNonce": 1761109632, "isDeleted": false, "id": "opI1A3GHk-glrkVHWtnzm", "fillStyle": "hachure", @@ -25811,7 +25811,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367072, "link": null, "locked": false, "fontSize": 19.993669122032355, @@ -25914,8 +25914,8 @@ }, { "type": "text", - "version": 406, - "versionNonce": 1003737164, + "version": 407, + "versionNonce": 235356544, "isDeleted": false, "id": "YeZy94byHPYDUDPEghHwz", "fillStyle": "hachure", @@ -25934,7 +25934,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367072, "link": null, "locked": false, "fontSize": 19.993669122032355, @@ -25986,8 +25986,8 @@ }, { "type": "text", - "version": 437, - "versionNonce": 2018060916, + "version": 438, + "versionNonce": 707237504, "isDeleted": false, "id": "oywRGTc7Qk21fMO5m7ipR", "fillStyle": "hachure", @@ -26006,7 +26006,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340740, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 19.99366912203235, @@ -26109,8 +26109,8 @@ }, { "type": "text", - "version": 469, - "versionNonce": 1812719308, + "version": 470, + "versionNonce": 514003328, "isDeleted": false, "id": "aKLeLpuGjjphQKAjHsJjd", "fillStyle": "hachure", @@ -26129,7 +26129,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 19.993669122032355, @@ -26393,8 +26393,8 @@ }, { "type": "text", - "version": 389, - "versionNonce": 404995060, + "version": 390, + "versionNonce": 1576060544, "isDeleted": false, "id": "-LC6sWVh5jazUS4DtR93c", "fillStyle": "hachure", @@ -26413,7 +26413,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 19.993669122032355, @@ -26479,8 +26479,8 @@ }, { "type": "text", - "version": 324, - "versionNonce": 972893516, + "version": 325, + "versionNonce": 1262805376, "isDeleted": false, "id": "2Y2E7wJPH9_gmr4i3S7pg", "fillStyle": "hachure", @@ -26499,7 +26499,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 19.993669122032355, @@ -26610,8 +26610,8 @@ }, { "type": "text", - "version": 480, - "versionNonce": 1370496372, + "version": 481, + "versionNonce": 554956416, "isDeleted": false, "id": "wR0GHC010wmNgfVtTA0EH", "fillStyle": "hachure", @@ -26635,7 +26635,7 @@ "type": "arrow" } ], - "updated": 1684138340741, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 19.993669122032355, @@ -26746,8 +26746,8 @@ }, { "type": "text", - "version": 746, - "versionNonce": 1996596172, + "version": 747, + "versionNonce": 590131584, "isDeleted": false, "id": "ZR7FlVxJKJhypbs2k28RF", "fillStyle": "hachure", @@ -26783,7 +26783,7 @@ "type": "arrow" } ], - "updated": 1684138340741, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 19.993669122032355, @@ -26945,8 +26945,8 @@ }, { "type": "text", - "version": 427, - "versionNonce": 775180020, + "version": 428, + "versionNonce": 2018584192, "isDeleted": false, "id": "KrxqLrVYs9q4JJfaCjd4e", "fillStyle": "hachure", @@ -26965,7 +26965,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 20, @@ -27008,8 +27008,8 @@ }, { "type": "text", - "version": 236, - "versionNonce": 959662668, + "version": 237, + "versionNonce": 1301971328, "isDeleted": false, "id": "h3exS8HSaQTUutxxP5KUd", "fillStyle": "hachure", @@ -27028,7 +27028,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367073, "link": null, "locked": false, "fontSize": 20, @@ -27076,8 +27076,8 @@ }, { "type": "text", - "version": 264, - "versionNonce": 506876020, + "version": 265, + "versionNonce": 1349015168, "isDeleted": false, "id": "sZwffdBtAx5k-pfeeH9pz", "fillStyle": "hachure", @@ -27096,7 +27096,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367074, "link": null, "locked": false, "fontSize": 20, @@ -27111,8 +27111,8 @@ }, { "type": "text", - "version": 197, - "versionNonce": 1131159756, + "version": 198, + "versionNonce": 164516224, "isDeleted": false, "id": "8VlmUF_Qt4ApUNPz0uyu9", "fillStyle": "hachure", @@ -27131,7 +27131,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367076, "link": null, "locked": false, "fontSize": 20, @@ -27234,8 +27234,8 @@ }, { "type": "text", - "version": 229, - "versionNonce": 293347828, + "version": 230, + "versionNonce": 2115602048, "isDeleted": false, "id": "osE6At5P-KA6Wkg8MGPZs", "fillStyle": "hachure", @@ -27254,7 +27254,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340741, + "updated": 1686229367076, "link": null, "locked": false, "fontSize": 20, @@ -27410,8 +27410,8 @@ }, { "type": "text", - "version": 347, - "versionNonce": 1833314124, + "version": 348, + "versionNonce": 789062016, "isDeleted": false, "id": "uCzkLthQbcSGl2d6NC343", "fillStyle": "hachure", @@ -27430,7 +27430,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367077, "link": null, "locked": false, "fontSize": 20, @@ -27445,8 +27445,8 @@ }, { "type": "text", - "version": 393, - "versionNonce": 252429172, + "version": 394, + "versionNonce": 1042593408, "isDeleted": false, "id": "DuEE3eExprGc4nGfb45In", "fillStyle": "hachure", @@ -27465,7 +27465,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367077, "link": null, "locked": false, "fontSize": 20, @@ -27508,8 +27508,8 @@ }, { "type": "text", - "version": 464, - "versionNonce": 186342860, + "version": 465, + "versionNonce": 747106688, "isDeleted": false, "id": "tUM3WeCUqPHDKsr80vNyx", "fillStyle": "hachure", @@ -27528,7 +27528,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367077, "link": null, "locked": false, "fontSize": 20, @@ -27576,8 +27576,8 @@ }, { "type": "text", - "version": 654, - "versionNonce": 1294677236, + "version": 655, + "versionNonce": 894035584, "isDeleted": false, "id": "Wj3gQoTmILHgxAxmQrr4Z", "fillStyle": "hachure", @@ -27596,7 +27596,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367077, "link": null, "locked": false, "fontSize": 20, @@ -27668,8 +27668,8 @@ }, { "type": "text", - "version": 379, - "versionNonce": 1443218508, + "version": 380, + "versionNonce": 160285056, "isDeleted": false, "id": "_m86-4rF1_MnbU0EYK2qV", "fillStyle": "hachure", @@ -27688,7 +27688,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367078, "link": null, "locked": false, "fontSize": 20, @@ -27881,8 +27881,8 @@ }, { "type": "text", - "version": 89, - "versionNonce": 1231058548, + "version": 90, + "versionNonce": 1883838080, "isDeleted": false, "id": "PrdiwQ3pRCa-rUhkk7yF9", "fillStyle": "hachure", @@ -27901,7 +27901,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367078, "link": null, "locked": false, "fontSize": 20, @@ -27949,8 +27949,8 @@ }, { "type": "text", - "version": 138, - "versionNonce": 1407146700, + "version": 139, + "versionNonce": 1700368768, "isDeleted": false, "id": "mLHLssqRUmGKiZ-GWX4rI", "fillStyle": "hachure", @@ -27969,7 +27969,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367080, "link": null, "locked": false, "fontSize": 20, @@ -28021,8 +28021,8 @@ }, { "type": "text", - "version": 183, - "versionNonce": 1618053108, + "version": 184, + "versionNonce": 1675773568, "isDeleted": false, "id": "JR2H-migNK_a2s1YcR1CR", "fillStyle": "hachure", @@ -28041,7 +28041,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367081, "link": null, "locked": false, "fontSize": 20, @@ -28238,8 +28238,8 @@ }, { "type": "text", - "version": 339, - "versionNonce": 1061298508, + "version": 340, + "versionNonce": 1999331712, "isDeleted": false, "id": "QYSVCDWaoUKpVYJomPC4S", "fillStyle": "hachure", @@ -28258,7 +28258,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367081, "link": null, "locked": false, "fontSize": 20, @@ -28306,8 +28306,8 @@ }, { "type": "text", - "version": 368, - "versionNonce": 1717856628, + "version": 369, + "versionNonce": 1825928832, "isDeleted": false, "id": "PflB2NVKX13-K9HwivQJL", "fillStyle": "hachure", @@ -28326,7 +28326,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340742, + "updated": 1686229367081, "link": null, "locked": false, "fontSize": 20, @@ -28378,8 +28378,8 @@ }, { "type": "text", - "version": 333, - "versionNonce": 1871189964, + "version": 334, + "versionNonce": 239284608, "isDeleted": false, "id": "nl4YAOoVhbSSjLCeOxUt7", "fillStyle": "hachure", @@ -28398,7 +28398,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367081, "link": null, "locked": false, "fontSize": 20, @@ -28446,8 +28446,8 @@ }, { "type": "text", - "version": 405, - "versionNonce": 1048517364, + "version": 406, + "versionNonce": 1340253824, "isDeleted": false, "id": "R_rqv4cLnmHd3kh8XwJoo", "fillStyle": "hachure", @@ -28466,7 +28466,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367081, "link": null, "locked": false, "fontSize": 20, @@ -28514,8 +28514,8 @@ }, { "type": "text", - "version": 407, - "versionNonce": 1087061580, + "version": 408, + "versionNonce": 299958656, "isDeleted": false, "id": "AGYI5oQ-SFXy_cFm8kSJ8", "fillStyle": "hachure", @@ -28534,7 +28534,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -28582,8 +28582,8 @@ }, { "type": "text", - "version": 388, - "versionNonce": 1934072948, + "version": 389, + "versionNonce": 969528960, "isDeleted": false, "id": "bwb7RbV_0WdKf0tH9vh64", "fillStyle": "hachure", @@ -28602,7 +28602,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -28650,8 +28650,8 @@ }, { "type": "text", - "version": 383, - "versionNonce": 2078643404, + "version": 384, + "versionNonce": 21287296, "isDeleted": false, "id": "toGGyuAwXBoFvomSM0kah", "fillStyle": "hachure", @@ -28670,7 +28670,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -28718,8 +28718,8 @@ }, { "type": "text", - "version": 395, - "versionNonce": 1317268980, + "version": 396, + "versionNonce": 1061881472, "isDeleted": false, "id": "16kUTRAS2Gir7zi1SR-5t", "fillStyle": "hachure", @@ -28738,7 +28738,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -28786,8 +28786,8 @@ }, { "type": "text", - "version": 375, - "versionNonce": 1047599948, + "version": 376, + "versionNonce": 1940824448, "isDeleted": false, "id": "bF4pX0avbTDgd01KzoJOZ", "fillStyle": "hachure", @@ -28806,7 +28806,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -28956,8 +28956,8 @@ }, { "type": "text", - "version": 315, - "versionNonce": 1899630452, + "version": 316, + "versionNonce": 1562785408, "isDeleted": false, "id": "YZ9v3JHi75hcJXtdWoC80", "fillStyle": "hachure", @@ -28976,7 +28976,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -29075,8 +29075,8 @@ }, { "type": "text", - "version": 191, - "versionNonce": 1657262540, + "version": 192, + "versionNonce": 408842624, "isDeleted": false, "id": "goZDhTXe0fEh9316nrqEW", "fillStyle": "hachure", @@ -29095,7 +29095,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -29147,8 +29147,8 @@ }, { "type": "text", - "version": 232, - "versionNonce": 1806696692, + "version": 233, + "versionNonce": 2015061632, "isDeleted": false, "id": "duy6YcR2fCrlkhrip8U1l", "fillStyle": "hachure", @@ -29167,7 +29167,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340743, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -29215,8 +29215,8 @@ }, { "type": "text", - "version": 277, - "versionNonce": 1745074252, + "version": 278, + "versionNonce": 915685760, "isDeleted": false, "id": "W4oOhlPUh7YsE9eciB0IO", "fillStyle": "hachure", @@ -29235,7 +29235,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367082, "link": null, "locked": false, "fontSize": 20, @@ -29436,8 +29436,8 @@ }, { "type": "text", - "version": 243, - "versionNonce": 1433125492, + "version": 244, + "versionNonce": 1558877824, "isDeleted": false, "id": "ou6j9SRzLx4OvDZx4PoZy", "fillStyle": "hachure", @@ -29456,7 +29456,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367083, "link": null, "locked": false, "fontSize": 20, @@ -29508,8 +29508,8 @@ }, { "type": "text", - "version": 303, - "versionNonce": 1509871308, + "version": 304, + "versionNonce": 764416384, "isDeleted": false, "id": "a8ImdV-blWOSy-Dr8Wlra", "fillStyle": "hachure", @@ -29528,7 +29528,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367083, "link": null, "locked": false, "fontSize": 20, @@ -29678,8 +29678,8 @@ }, { "type": "text", - "version": 365, - "versionNonce": 1697862644, + "version": 366, + "versionNonce": 1079232128, "isDeleted": false, "id": "cV894oOyFNUZ1t-cSR7wC", "fillStyle": "hachure", @@ -29698,7 +29698,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367084, "link": null, "locked": false, "fontSize": 20, @@ -29746,8 +29746,8 @@ }, { "type": "text", - "version": 400, - "versionNonce": 961746252, + "version": 401, + "versionNonce": 1955717504, "isDeleted": false, "id": "5J_CNG1ap2xlpzJj6xL6_", "fillStyle": "hachure", @@ -29766,7 +29766,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367084, "link": null, "locked": false, "fontSize": 20, @@ -29814,8 +29814,8 @@ }, { "type": "text", - "version": 338, - "versionNonce": 151827828, + "version": 339, + "versionNonce": 1058469504, "isDeleted": false, "id": "9M9W-Y3h_czJnunELhQ5l", "fillStyle": "hachure", @@ -29834,7 +29834,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367085, "link": null, "locked": false, "fontSize": 20, @@ -29882,8 +29882,8 @@ }, { "type": "text", - "version": 400, - "versionNonce": 1666878412, + "version": 401, + "versionNonce": 450474368, "isDeleted": false, "id": "OSqoS9olz_y2c3UALew8A", "fillStyle": "hachure", @@ -29902,7 +29902,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367086, "link": null, "locked": false, "fontSize": 20, @@ -29945,8 +29945,8 @@ }, { "type": "text", - "version": 455, - "versionNonce": 2117986036, + "version": 456, + "versionNonce": 1576281728, "isDeleted": false, "id": "K63fthKq1zDCMVb_JfTxI", "fillStyle": "hachure", @@ -29965,7 +29965,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367086, "link": null, "locked": false, "fontSize": 20, @@ -30008,8 +30008,8 @@ }, { "type": "text", - "version": 461, - "versionNonce": 2049913420, + "version": 462, + "versionNonce": 1202160000, "isDeleted": false, "id": "SZx434IEAyN8u9Iok9wKs", "fillStyle": "hachure", @@ -30028,7 +30028,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367087, "link": null, "locked": false, "fontSize": 20, @@ -30071,8 +30071,8 @@ }, { "type": "text", - "version": 454, - "versionNonce": 1766647924, + "version": 455, + "versionNonce": 162223744, "isDeleted": false, "id": "PXg6Txuhjr9XnACRPA3o1", "fillStyle": "hachure", @@ -30091,7 +30091,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367087, "link": null, "locked": false, "fontSize": 20, @@ -30134,8 +30134,8 @@ }, { "type": "text", - "version": 136, - "versionNonce": 1286277324, + "version": 137, + "versionNonce": 976966016, "isDeleted": false, "id": "PsCFAZngKhjlM8aBiROp_", "fillStyle": "hachure", @@ -30154,7 +30154,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340744, + "updated": 1686229367087, "link": null, "locked": false, "fontSize": 20, @@ -30337,8 +30337,8 @@ }, { "type": "text", - "version": 412, - "versionNonce": 1845475828, + "version": 413, + "versionNonce": 385648256, "isDeleted": false, "id": "I3IyBpdIHc46DQR-k_9ug", "fillStyle": "hachure", @@ -30357,7 +30357,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367087, "link": null, "locked": false, "fontSize": 20, @@ -30428,8 +30428,8 @@ }, { "type": "text", - "version": 408, - "versionNonce": 1047007052, + "version": 409, + "versionNonce": 1238704512, "isDeleted": false, "id": "DAsVQ52b2NSGbWD0WScVx", "fillStyle": "hachure", @@ -30448,7 +30448,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367088, "link": null, "locked": false, "fontSize": 20, @@ -30463,8 +30463,8 @@ }, { "type": "text", - "version": 374, - "versionNonce": 1197235060, + "version": 375, + "versionNonce": 1118287488, "isDeleted": false, "id": "QZ18SBPUnlBTjUs67xOhc", "fillStyle": "hachure", @@ -30483,7 +30483,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367088, "link": null, "locked": false, "fontSize": 20, @@ -30547,8 +30547,8 @@ }, { "type": "text", - "version": 401, - "versionNonce": 801129932, + "version": 402, + "versionNonce": 1706357120, "isDeleted": false, "id": "cIbePAL0iwRYpxURuJ-PN", "fillStyle": "hachure", @@ -30567,7 +30567,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367088, "link": null, "locked": false, "fontSize": 20, @@ -30634,8 +30634,8 @@ }, { "type": "text", - "version": 379, - "versionNonce": 1604898036, + "version": 380, + "versionNonce": 829220480, "isDeleted": false, "id": "5skfbsSo3bMF02GgWaVH7", "fillStyle": "hachure", @@ -30654,7 +30654,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367088, "link": null, "locked": false, "fontSize": 20, @@ -30697,8 +30697,8 @@ }, { "type": "text", - "version": 382, - "versionNonce": 1707977804, + "version": 383, + "versionNonce": 354075008, "isDeleted": false, "id": "cwQfv7ctKwc9rzmOEdeLR", "fillStyle": "hachure", @@ -30717,7 +30717,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367088, "link": null, "locked": false, "fontSize": 20, @@ -30900,8 +30900,8 @@ }, { "type": "text", - "version": 338, - "versionNonce": 1751846516, + "version": 339, + "versionNonce": 1879840384, "isDeleted": false, "id": "rCH1-6E8SS-myByDCU3hi", "fillStyle": "hachure", @@ -30920,7 +30920,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -30935,8 +30935,8 @@ }, { "type": "text", - "version": 259, - "versionNonce": 1919566540, + "version": 260, + "versionNonce": 2001113472, "isDeleted": false, "id": "sIQsIbPKVe0sTNSdQqzJp", "fillStyle": "hachure", @@ -30955,7 +30955,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31003,8 +31003,8 @@ }, { "type": "text", - "version": 43, - "versionNonce": 1919492084, + "version": 44, + "versionNonce": 1933919872, "isDeleted": false, "id": "AQ-S-qacCfvyfW8sMEBB8", "fillStyle": "hachure", @@ -31023,7 +31023,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31071,8 +31071,8 @@ }, { "type": "text", - "version": 105, - "versionNonce": 472910156, + "version": 106, + "versionNonce": 1131962752, "isDeleted": false, "id": "FRfO_QmhI2jnLy0GRu1Bn", "fillStyle": "hachure", @@ -31091,7 +31091,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340745, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31139,8 +31139,8 @@ }, { "type": "text", - "version": 50, - "versionNonce": 94962036, + "version": 51, + "versionNonce": 400062080, "isDeleted": false, "id": "KoFu6DwedxvPDILTvJYxO", "fillStyle": "hachure", @@ -31159,7 +31159,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340746, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31207,8 +31207,8 @@ }, { "type": "text", - "version": 44, - "versionNonce": 1182334924, + "version": 45, + "versionNonce": 1223700864, "isDeleted": false, "id": "RsA7-KvsaIssMv6I9zEa_", "fillStyle": "hachure", @@ -31227,7 +31227,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340746, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31275,8 +31275,8 @@ }, { "type": "text", - "version": 33, - "versionNonce": 963687156, + "version": 34, + "versionNonce": 579184256, "isDeleted": false, "id": "26gvJ4inCvQL7fRm3uTMU", "fillStyle": "hachure", @@ -31295,7 +31295,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340746, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31343,8 +31343,8 @@ }, { "type": "text", - "version": 37, - "versionNonce": 1499407948, + "version": 38, + "versionNonce": 1861091712, "isDeleted": false, "id": "d8AadxKck8R8lIHl_GGhH", "fillStyle": "hachure", @@ -31363,7 +31363,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340746, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31415,8 +31415,8 @@ }, { "type": "text", - "version": 406, - "versionNonce": 1279669364, + "version": 407, + "versionNonce": 1074583168, "isDeleted": false, "id": "c4716TqeE5DN-SNsNd9S8", "fillStyle": "hachure", @@ -31435,7 +31435,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340746, + "updated": 1686229367089, "link": null, "locked": false, "fontSize": 20, @@ -31483,8 +31483,8 @@ }, { "type": "text", - "version": 441, - "versionNonce": 700218572, + "version": 442, + "versionNonce": 884068736, "isDeleted": false, "id": "eXG0QxF7KLgxZeFehHc8P", "fillStyle": "hachure", @@ -31503,7 +31503,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340746, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 20, @@ -31555,8 +31555,8 @@ }, { "type": "text", - "version": 498, - "versionNonce": 1005553140, + "version": 499, + "versionNonce": 86902400, "isDeleted": false, "id": "e2bSHJqVFym72u_0aWncY", "fillStyle": "hachure", @@ -31575,7 +31575,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340746, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 20, @@ -31627,8 +31627,8 @@ }, { "type": "text", - "version": 543, - "versionNonce": 1630208844, + "version": 544, + "versionNonce": 830185856, "isDeleted": false, "id": "0O84_qw7tqC7CEAteDbzH", "fillStyle": "hachure", @@ -31647,7 +31647,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340747, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 20, @@ -31695,8 +31695,8 @@ }, { "type": "text", - "version": 503, - "versionNonce": 1567795060, + "version": 504, + "versionNonce": 1856583296, "isDeleted": false, "id": "r95mtlZ6udx6KoasYfPP-", "fillStyle": "hachure", @@ -31715,7 +31715,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340747, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 20, @@ -31818,8 +31818,8 @@ }, { "type": "text", - "version": 616, - "versionNonce": 1711585740, + "version": 617, + "versionNonce": 344682880, "isDeleted": false, "id": "TdXEUn2RLq8C0tjp0P_mI", "fillStyle": "hachure", @@ -31838,7 +31838,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340747, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 20, @@ -32464,8 +32464,8 @@ }, { "type": "text", - "version": 59, - "versionNonce": 1964349684, + "version": 60, + "versionNonce": 1631479424, "isDeleted": false, "id": "IZmYbAvhwLOI-Fvyn9Eea", "fillStyle": "hachure", @@ -32484,7 +32484,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340747, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 20, @@ -32671,8 +32671,8 @@ }, { "type": "text", - "version": 121, - "versionNonce": 1130602572, + "version": 122, + "versionNonce": 622936448, "isDeleted": false, "id": "60rTmjiMEDvJ_9hKBmY8v", "fillStyle": "hachure", @@ -32691,7 +32691,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340747, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 20, @@ -32884,8 +32884,8 @@ }, { "type": "text", - "version": 326, - "versionNonce": 1314806388, + "version": 327, + "versionNonce": 699242112, "isDeleted": false, "id": "JGzteL9meg3s3V76x17P5", "fillStyle": "hachure", @@ -32904,7 +32904,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340747, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 28, @@ -32919,8 +32919,8 @@ }, { "type": "text", - "version": 612, - "versionNonce": 287422156, + "version": 613, + "versionNonce": 1115492736, "isDeleted": false, "id": "c6ZdG-H-hh_aHr6l1y-JA", "fillStyle": "hachure", @@ -32939,7 +32939,7 @@ "groupIds": [], "roundness": null, "boundElements": [], - "updated": 1684138340747, + "updated": 1686229367090, "link": null, "locked": false, "fontSize": 28, diff --git a/test/setup/create-cluster-resources.sh b/test/setup/create-cluster-resources.sh old mode 100644 new mode 100755 index 90b0bf01..27266be3 --- a/test/setup/create-cluster-resources.sh +++ b/test/setup/create-cluster-resources.sh @@ -5,6 +5,10 @@ CLUSTER_NAME=kubehound.test.local CONFIG_DIR=./test-cluster echo "[*] Deploying test resources via kubectl apply" -kubectl apply -f "${CONFIG_DIR}/priv-pod.yaml" --context "kind-${CLUSTER_NAME}" -kubectl apply -f "${CONFIG_DIR}/priv-pid-pod.yaml" --context "kind-${CLUSTER_NAME}" -kubectl apply -f "${CONFIG_DIR}/hostpath-pod.yaml" --context "kind-${CLUSTER_NAME}" +for attack in ${CONFIG_DIR}/attacks/*.yaml; do + [ -e "$attack" ] || continue + + kubectl apply -f "$attack" --context "kind-${CLUSTER_NAME}" +done + +echo "[*] Deployments complete" \ No newline at end of file diff --git a/test/setup/test-cluster/attacks/CE_HOST_MOUNT.yaml b/test/setup/test-cluster/attacks/CE_HOST_MOUNT.yaml new file mode 100644 index 00000000..e3a9fc9e --- /dev/null +++ b/test/setup/test-cluster/attacks/CE_HOST_MOUNT.yaml @@ -0,0 +1,16 @@ +# CE_HOST_MOUNT edge +# @@DOCLINK: TODO TOOD see https://github.com/BishopFox/badPods/tree/main/manifests/priv#mount-the-hosts-filesystem +apiVersion: v1 +kind: Pod +metadata: + name: priv-pod + labels: + app: kubehound-edge-test +spec: + containers: + - name: priv-pod + image: ubuntu + securityContext: + privileged: true + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/CE_MODULE_LOAD.yaml b/test/setup/test-cluster/attacks/CE_MODULE_LOAD.yaml new file mode 100644 index 00000000..e4334899 --- /dev/null +++ b/test/setup/test-cluster/attacks/CE_MODULE_LOAD.yaml @@ -0,0 +1,17 @@ +# CE_MODULE_LOAD edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2890006884/CE+MODULE+LOAD +apiVersion: v1 +kind: Pod +metadata: + name: modload-pod + labels: + app: kubehound-edge-test +spec: + containers: + - name: modload-pod + image: ubuntu + securityContext: + capabilities: + add: ["SYS_MODULE"] + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/CE_NET_MITM.yaml b/test/setup/test-cluster/attacks/CE_NET_MITM.yaml new file mode 100644 index 00000000..654463a6 --- /dev/null +++ b/test/setup/test-cluster/attacks/CE_NET_MITM.yaml @@ -0,0 +1,17 @@ +# CE_NET_ADMIN edge +# @@DOCLINK: TODO TODO +apiVersion: v1 +kind: Pod +metadata: + name: netadmin-pod + labels: + app: kubehound-edge-test +spec: + containers: + - name: netadmin-pod + image: ubuntu + securityContext: + capabilities: + add: ["NET_ADMIN"] + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/CE_NSENTER.yaml b/test/setup/test-cluster/attacks/CE_NSENTER.yaml new file mode 100644 index 00000000..f33191ef --- /dev/null +++ b/test/setup/test-cluster/attacks/CE_NSENTER.yaml @@ -0,0 +1,17 @@ +# CE_NS_ENTER edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2897872074/CE+NSENTER +apiVersion: v1 +kind: Pod +metadata: + name: nsenter-pod + labels: + app: kubehound-edge-test +spec: + hostPID: true + containers: + - name: nsenter-pod + image: ubuntu + securityContext: + privileged: true + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/CE_UMH_CORE_PATTERN.yaml b/test/setup/test-cluster/attacks/CE_UMH_CORE_PATTERN.yaml new file mode 100644 index 00000000..02d50b3c --- /dev/null +++ b/test/setup/test-cluster/attacks/CE_UMH_CORE_PATTERN.yaml @@ -0,0 +1,21 @@ +# CE_UMH_CORE_PATTERN edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2889843064/CE+UMH+CORE+PATTERN +apiVersion: v1 +kind: Pod +metadata: + name: umh-core-pod + labels: + app: kubehound-edge-test +spec: + containers: + - name: umh-core-pod + image: ubuntu + volumeMounts: + - mountPath: /sysproc + name: nodeproc + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] + volumes: + - name: nodeproc + hostPath: + path: /proc/sys/kernel diff --git a/test/setup/test-cluster/attacks/IDENTITY_IMPERSONATE.yaml b/test/setup/test-cluster/attacks/IDENTITY_IMPERSONATE.yaml new file mode 100644 index 00000000..7a38ad15 --- /dev/null +++ b/test/setup/test-cluster/attacks/IDENTITY_IMPERSONATE.yaml @@ -0,0 +1,45 @@ +# IDENTITY_IMPERSONATE edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2887123111/IDENTITY+IMPERSONATE +apiVersion: v1 +kind: ServiceAccount +metadata: + name: impersonate-sa + namespace: default +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: impersonate +rules: + - apiGroups: ["*"] + resources: ["users", "groups"] + verbs: ["impersonate"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-impersonate + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: impersonate +subjects: + - kind: ServiceAccount + name: impersonate-sa + namespace: default +--- +apiVersion: v1 +kind: Pod +metadata: + name: impersonate-pod + labels: + app: kubehound-edge-test +spec: + serviceAccountName: impersonate-sa + containers: + - name: impersonate-pod + image: ubuntu + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/POD_CREATE.yaml b/test/setup/test-cluster/attacks/POD_CREATE.yaml new file mode 100644 index 00000000..386cbb74 --- /dev/null +++ b/test/setup/test-cluster/attacks/POD_CREATE.yaml @@ -0,0 +1,46 @@ +# POD_CREATE_PRIV / POD_SCHEDULE_NODE +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2885716055/POD+CREATE+PRIV +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2886370480/POD+SCHEDULE+NODE +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pod-create-sa + namespace: default +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: create-pods +rules: + - apiGroups: ["*"] + resources: ["pods"] + verbs: ["get", "list", "create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-create-secrets + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: create-pods +subjects: + - kind: ServiceAccount + name: pod-create-sa + namespace: default +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-create-pod + labels: + app: kubehound-edge-test +spec: + serviceAccountName: pod-create-sa + containers: + - name: pod-create-pod + image: ubuntu + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/POD_PATCH.yaml b/test/setup/test-cluster/attacks/POD_PATCH.yaml new file mode 100644 index 00000000..dfa17445 --- /dev/null +++ b/test/setup/test-cluster/attacks/POD_PATCH.yaml @@ -0,0 +1,45 @@ +# POD_PATCH edge +# @@DOCLINK: TODO TODO +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pod-patch-sa + namespace: default +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: patch-pods +rules: + - apiGroups: ["*"] + resources: ["pods"] + verbs: ["get", "list", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-patch-pods + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: patch-pods +subjects: + - kind: ServiceAccount + name: pod-patch-sa + namespace: default +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-patch-pod + labels: + app: kubehound-edge-test +spec: + serviceAccountName: pod-patch-sa + containers: + - name: pod-patch-pod + image: ubuntu + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/ROLE_BIND.yaml b/test/setup/test-cluster/attacks/ROLE_BIND.yaml new file mode 100644 index 00000000..d5f04074 --- /dev/null +++ b/test/setup/test-cluster/attacks/ROLE_BIND.yaml @@ -0,0 +1,48 @@ +# ROLE_BIND edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2889355675/ROLE+BIND +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rolebind-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rolebind +rules: + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["create"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "roles"] + verbs: ["bind"] + resourceNames: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-bind-role + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rolebind +subjects: + - kind: ServiceAccount + name: rolebind-sa + namespace: default +--- +apiVersion: v1 +kind: Pod +metadata: + name: rolebind-pod + labels: + app: kubehound-edge-test +spec: + serviceAccountName: rolebind-sa + containers: + - name: rolebind-pod + image: ubuntu + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/SHARED_PS_NAMESPACE.yaml b/test/setup/test-cluster/attacks/SHARED_PS_NAMESPACE.yaml new file mode 100644 index 00000000..4ed84a36 --- /dev/null +++ b/test/setup/test-cluster/attacks/SHARED_PS_NAMESPACE.yaml @@ -0,0 +1,15 @@ +# SHARED_PS_NAMESPACE edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2880275294/SHARED+PS+NAMESPACE +apiVersion: v1 +kind: Pod +metadata: + name: sharedps-pod + labels: + app: kubehound-edge-test +spec: + shareProcessNamespace: true + containers: + - name: sharedps-pod + image: ubuntu + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/TOKEN_BRUTEFORCE.yaml b/test/setup/test-cluster/attacks/TOKEN_BRUTEFORCE.yaml new file mode 100644 index 00000000..8924cbe9 --- /dev/null +++ b/test/setup/test-cluster/attacks/TOKEN_BRUTEFORCE.yaml @@ -0,0 +1,45 @@ +# TOKEN_BRUTEFORCE edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2887155994/TOKEN+BRUTEFORCE +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tokenget-sa + namespace: default +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: read-secrets +rules: + - apiGroups: ["*"] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-get-secrets + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: read-secrets +subjects: + - kind: ServiceAccount + name: tokenget-sa + namespace: default +--- +apiVersion: v1 +kind: Pod +metadata: + name: tokenget-pod + labels: + app: kubehound-edge-test +spec: + serviceAccountName: tokenget-sa + containers: + - name: tokenget-pod + image: ubuntu + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/TOKEN_LIST.yaml b/test/setup/test-cluster/attacks/TOKEN_LIST.yaml new file mode 100644 index 00000000..832cb3a7 --- /dev/null +++ b/test/setup/test-cluster/attacks/TOKEN_LIST.yaml @@ -0,0 +1,45 @@ +# TOKEN_LIST edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2886795639/TOKEN+LIST +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tokenlist-sa + namespace: default +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default + name: list-secrets +rules: + - apiGroups: ["*"] + resources: ["secrets"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-list-secrets + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: list-secrets +subjects: + - kind: ServiceAccount + name: tokenlist-sa + namespace: default +--- +apiVersion: v1 +kind: Pod +metadata: + name: tokenlist-pod + labels: + app: kubehound-edge-test +spec: + serviceAccountName: tokenlist-sa + containers: + - name: tokenlist-pod + image: ubuntu + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/test/setup/test-cluster/attacks/TOKEN_VAR_LOG_SYMLINK.yaml b/test/setup/test-cluster/attacks/TOKEN_VAR_LOG_SYMLINK.yaml new file mode 100644 index 00000000..e532cd71 --- /dev/null +++ b/test/setup/test-cluster/attacks/TOKEN_VAR_LOG_SYMLINK.yaml @@ -0,0 +1,21 @@ +# TOKEN_VAR_LOG_SYMLINK edge +# @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2895282182/TOKEN+VAR+LOG+SYMLINK +apiVersion: v1 +kind: Pod +metadata: + name: varlog-pod + labels: + app: kubehound-edge-test +spec: + containers: + - name: varlog-pod + image: ubuntu + volumeMounts: + - mountPath: /host/var/log + name: nodelog + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] + volumes: + - name: nodelog + hostPath: + path: /var/log diff --git a/test/setup/test-cluster/hostpath-pod.yaml b/test/setup/test-cluster/hostpath-pod.yaml deleted file mode 100644 index ec360c85..00000000 --- a/test/setup/test-cluster/hostpath-pod.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: hostpath-pod - labels: - app: kubehound-edge-test -spec: - containers: - - name: hostpath-pod - image: ubuntu - volumeMounts: - - mountPath: /host - name: noderoot - command: [ "/bin/sh", "-c", "--" ] - args: [ "while true; do sleep 30; done;" ] - volumes: - - name: noderoot - hostPath: - path: / \ No newline at end of file diff --git a/test/setup/test-cluster/priv-pid-pod.yaml b/test/setup/test-cluster/priv-pid-pod.yaml deleted file mode 100644 index 2ff311d7..00000000 --- a/test/setup/test-cluster/priv-pid-pod.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: priv-hostpid-pod - labels: - app: kubehound-edge-test -spec: - hostPID: true - containers: - - name: priv-hostpid-pod - image: ubuntu - tty: true - securityContext: - privileged: true - command: [ "nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--net", "--pid", "--", "bash" ] diff --git a/test/setup/test-cluster/priv-pod.yaml b/test/setup/test-cluster/priv-pod.yaml deleted file mode 100644 index d14c8542..00000000 --- a/test/setup/test-cluster/priv-pod.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: priv-pod - labels: - app: kubehound-edge-test -spec: - containers: - - name: priv-pod - image: ubuntu - securityContext: - privileged: true - command: [ "/bin/sh", "-c", "--" ] - args: [ "while true; do sleep 30; done;" ] From 79a68f7c2575fdb88a3cadd6ab53763d02a08ddf Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 10:21:06 +0200 Subject: [PATCH 25/61] remove compromised --- pkg/kubehound/graph/vertex/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index bae4de11..508587db 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -84,7 +84,7 @@ func (v Container) Traversal() VertexTraversal { // Property(gremlingo.Cardinality.Set, "ports", gremlingo.T__.Select("c").Select("ports")). Property("pod", gremlingo.T__.Select("c").Select("pod")). Property("node", gremlingo.T__.Select("c").Select("node")). - Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). Property("critical", gremlingo.T__.Select("c").Select("critical")) return traversal From 18167091c886574e0053f42da840c4007d8a3956 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 11:31:24 +0200 Subject: [PATCH 26/61] aled --- pkg/kubehound/graph/vertex/container.go | 64 ++++++++++++------- pkg/kubehound/graph/vertex/identity.go | 34 +++++++--- pkg/kubehound/graph/vertex/node.go | 59 +++++++++++++---- pkg/kubehound/graph/vertex/pod.go | 38 +++++++---- pkg/kubehound/graph/vertex/role.go | 32 ++++++---- pkg/kubehound/graph/vertex/volume.go | 29 ++++++--- .../storage/graphdb/janusgraph_writer.go | 1 + pkg/utils/interfaces.go | 14 ++++ 8 files changed, 192 insertions(+), 79 deletions(-) diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 508587db..8992fb8e 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -25,7 +25,7 @@ func (v Container) BatchSize() int { } func (v Container) Traversal() VertexTraversal { - return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { + return func(source *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { // g = g.GetGraphTraversal() // for _, insert := range inserts { @@ -66,27 +66,45 @@ func (v Container) Traversal() VertexTraversal { // return g.GetGraphTraversal() // I have no idea how I can convert the arrays in the inserts to multiple values with cardinality set.... insertsConverted := utils.ConvertSliceAnyToTyped[graph.Container, TraversalInput](inserts) - log.I.Infof(" ============== INSERTS Containers ====== %+v", insertsConverted) - traversal := g.Inject(insertsConverted).Unfold().As("c"). - AddV(v.Label()). - Property("storeId", gremlingo.T__.Select("c").Select("store_id")). - Property("name", gremlingo.T__.Select("c").Select("name")). - Property("image", gremlingo.T__.Select("c").Select("image")). - // Property(gremlingo.Cardinality.Set, "command", gremlingo.T__.Select("c").Select("command")). - // Property(gremlingo.Cardinality.Set, "args", gremlingo.T__.Select("c").Select("args")). - // Property(gremlingo.Cardinality.Set, "capabilities", gremlingo.T__.Select("c").Select("capabilities")). - Property("privileged", gremlingo.T__.Select("c").Select("privileged")). - Property("privesc", gremlingo.T__.Select("c").Select("privesc")). - Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). - Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). - Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). - Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). - // Property(gremlingo.Cardinality.Set, "ports", gremlingo.T__.Select("c").Select("ports")). - Property("pod", gremlingo.T__.Select("c").Select("pod")). - Property("node", gremlingo.T__.Select("c").Select("node")). - // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - Property("critical", gremlingo.T__.Select("c").Select("critical")) - - return traversal + toStore := utils.ConvertToSliceMapAny(insertsConverted) + log.I.Infof(" ============== INSERTS Container ====== %+v", insertsConverted) + log.I.Infof(" ============== toStore Container ====== %+v", toStore) + g := source.GetGraphTraversal() + for _, i := range toStore { + g = g.AddV(v.Label()). + Property("storeId", i["storeId"]). + Property("name", i["name"]). + Property("image", i["image"]). + Property("privileged", i["privileged"]). + Property("privesc", i["privesc"]). + Property("hostPid", i["hostPid"]). + Property("hostPath", i["hostPath"]). + Property("hostNetwork", i["hostNetwork"]). + Property("runAsUser", i["runAsUser"]). + Property("pod", i["pod"]). + Property("node", i["node"]). + // Property("compromised", i["compromised"]). + Property("critical", i["critical"]) + } + return g + // return g.Inject(toStore).Unfold().As("c"). + // AddV(v.Label()). + // Property("storeId", gremlingo.T__.Select("c").Select("store_id")). + // Property("name", gremlingo.T__.Select("c").Select("name")). + // Property("image", gremlingo.T__.Select("c").Select("image")). + // // Property(gremlingo.Cardinality.Set, "command", gremlingo.T__.Select("c").Select("command")). + // // Property(gremlingo.Cardinality.Set, "args", gremlingo.T__.Select("c").Select("args")). + // // Property(gremlingo.Cardinality.Set, "capabilities", gremlingo.T__.Select("c").Select("capabilities")). + // Property("privileged", gremlingo.T__.Select("c").Select("privileged")). + // Property("privesc", gremlingo.T__.Select("c").Select("privesc")). + // Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). + // Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). + // Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). + // Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). + // // Property(gremlingo.Cardinality.Set, "ports", gremlingo.T__.Select("c").Select("ports")). + // Property("pod", gremlingo.T__.Select("c").Select("pod")). + // Property("node", gremlingo.T__.Select("c").Select("node")). + // // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + // Property("critical", gremlingo.T__.Select("c").Select("critical")) } } diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index f450e230..8009dd7c 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -25,15 +25,31 @@ func (v Identity) BatchSize() int { } func (v Identity) Traversal() VertexTraversal { - return func(g *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { + return func(source *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { insertsConverted := utils.ConvertSliceAnyToTyped[graph.Identity, TraversalInput](inserts) - log.I.Infof(" ============== INSERTS Identities ====== %+v", insertsConverted) - return g.Inject(inserts).Unfold().As("c"). - AddV(v.Label()). - Property("storeId", gremlingo.T__.Select("c").Select("store_id")). - Property("name", gremlingo.T__.Select("c").Select("name")). - Property("isNamespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - Property("namespace", gremlingo.T__.Select("c").Select("namespace")). - Property("type", gremlingo.T__.Select("c").Select("type")) + toStore := utils.ConvertToSliceMapAny(insertsConverted) + log.I.Infof(" ============== INSERTS Identity ====== %+v", insertsConverted) + log.I.Infof(" ============== toStore Identity ====== %+v", toStore) + g := source.GetGraphTraversal() + for _, i := range toStore { + // i := insert.(map[string]any) + // command := utils.ToSliceOfAny(i["command"].([]any)) + // args := utils.ToSliceOfAny(i["args"].([]any)) + // capabilities := utils.ToSliceOfAny(i["capabilities"].([]any)) + + g = g.AddV(v.Label()). + Property("storeId", i["storeId"]). + Property("name", i["name"]). + Property("isNamespaced", i["isNamespaced"]). + Property("namespace", i["namespace"]). + Property("type", i["type"]) + } + return g + // return g.Inject(toStore).Unfold().As("c").AddV(v.Label()). + // Property("storeId", gremlingo.T__.Select("c").Select("store_id")). + // Property("name", gremlingo.T__.Select("c").Select("name")). + // Property("isNamespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + // Property("namespace", gremlingo.T__.Select("c").Select("namespace")). + // Property("type", gremlingo.T__.Select("c").Select("type")) } } diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 5fdc3a7b..1fe0227b 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -5,7 +5,6 @@ import ( "github.com/DataDog/KubeHound/pkg/telemetry/log" "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" - gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) const ( @@ -26,17 +25,55 @@ func (v Node) BatchSize() int { } func (v Node) Traversal() VertexTraversal { - return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { insertsConverted := utils.ConvertSliceAnyToTyped[graph.Node, TraversalInput](inserts) + toStore := utils.ConvertToSliceMapAny(insertsConverted) log.I.Infof(" ============== INSERTS Nodes ====== %+v", insertsConverted) - traversal := g.Inject(inserts).Unfold().As("c"). - AddV(v.Label()). - Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - Property("name", gremlingo.T__.Select("c").Select("name")). - Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - Property("namespace", gremlingo.T__.Select("c").Select("namespace")). - Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - Property("critical", gremlingo.T__.Select("c").Select("critical")) - return traversal + log.I.Infof(" ============== toStore Nodes ====== %+v", toStore) + g := source.GetGraphTraversal() + for _, i := range toStore { + // i := insert.(map[string]any) + // command := utils.ToSliceOfAny(i["command"].([]any)) + // args := utils.ToSliceOfAny(i["args"].([]any)) + // capabilities := utils.ToSliceOfAny(i["capabilities"].([]any)) + + g = g.AddV(v.Label()). + Property("storeId", i["storeId"]). + Property("name", i["name"]). + Property("image", i["image"]). + Property("privileged", i["privileged"]). + Property("privesc", i["privesc"]). + Property("hostPid", i["hostPid"]). + Property("hostPath", i["hostPath"]). + Property("hostNetwork", i["hostNetwork"]). + Property("runAsUser", i["runAsUser"]). + Property("pod", i["pod"]). + Property("node", i["node"]). + // Property("compromised", i["compromised"]). + Property("critical", i["critical"]) + + // for _, cmd := range command { + // traversal = traversal.Property(gremlingo.Cardinality.Set, "command", cmd) + // } + // for _, arg := range args { + // traversal = traversal.Property(gremlingo.Cardinality.Set, "args", arg) + // } + // for _, cap := range capabilities { + // traversal = traversal.Property(gremlingo.Cardinality.Set, "capabilities", cap) + // } + // for _, port := range ports { + // traversal = traversal.Property(gremlingo.Cardinality.Set, "ports", port) + // } + } + return g + // return g.Inject(toStore).Unfold().As("c"). + // AddV(v.Label()). + // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + // Property("name", gremlingo.T__.Select("c").Select("name")). + // Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + // Property("namespace", gremlingo.T__.Select("c").Select("namespace")). + // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + // Property("critical", gremlingo.T__.Select("c").Select("critical")) + } } diff --git a/pkg/kubehound/graph/vertex/pod.go b/pkg/kubehound/graph/vertex/pod.go index 46f6bf5c..942d7afc 100644 --- a/pkg/kubehound/graph/vertex/pod.go +++ b/pkg/kubehound/graph/vertex/pod.go @@ -5,7 +5,6 @@ import ( "github.com/DataDog/KubeHound/pkg/telemetry/log" "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" - gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) const ( @@ -26,20 +25,31 @@ func (v Pod) BatchSize() int { } func (v Pod) Traversal() VertexTraversal { - return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { insertsConverted := utils.ConvertSliceAnyToTyped[graph.Pod, TraversalInput](inserts) + toStore := utils.ConvertToSliceMapAny(insertsConverted) log.I.Infof(" ============== INSERTS Pods ====== %+v", insertsConverted) - traversal := g.Inject(inserts).Unfold().As("c"). - AddV(v.Label()). - Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - Property("name", gremlingo.T__.Select("c").Select("name")). - Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - Property("namespace", gremlingo.T__.Select("c").Select("namespace")). - Property("sharedProcessNamespace", gremlingo.T__.Select("c").Select("sharedProcessNamespace")). - Property("serviceAccount", gremlingo.T__.Select("c").Select("serviceAccount")). - Property("node", gremlingo.T__.Select("c").Select("node")). - Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - Property("critical", gremlingo.T__.Select("c").Select("critical")) - return traversal + log.I.Infof(" ============== toStore Pods ====== %+v", toStore) + g := source.GetGraphTraversal() + for _, i := range toStore { + g = g.AddV(v.Label()). + Property("storeId", i["storeId"]). + Property("name", i["name"]). + Property("isNamespaced", i["isNamespaced"]). + Property("namespace", i["namespace"]). + Property("type", i["type"]) + } + return g + // return g.Inject(toStore).Unfold().As("c"). + // AddV(v.Label()). + // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + // Property("name", gremlingo.T__.Select("c").Select("name")). + // Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + // Property("namespace", gremlingo.T__.Select("c").Select("namespace")). + // Property("sharedProcessNamespace", gremlingo.T__.Select("c").Select("sharedProcessNamespace")). + // Property("serviceAccount", gremlingo.T__.Select("c").Select("serviceAccount")). + // Property("node", gremlingo.T__.Select("c").Select("node")). + // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). + // Property("critical", gremlingo.T__.Select("c").Select("critical")) } } diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index 89824713..b03c472e 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -5,7 +5,6 @@ import ( "github.com/DataDog/KubeHound/pkg/telemetry/log" "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" - gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) const ( @@ -26,17 +25,26 @@ func (v Role) BatchSize() int { } func (v Role) Traversal() VertexTraversal { - return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { insertsConverted := utils.ConvertSliceAnyToTyped[graph.Role, TraversalInput](inserts) - log.I.Infof(" ============== INSERTS ROLES ====== %+v", insertsConverted) - - traversal := g.Inject(insertsConverted).Unfold().As("c"). - AddV(v.Label()). - Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - Property("name", gremlingo.T__.Select("c").Select("name")). - Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - Property("namespace", gremlingo.T__.Select("c").Select("namespace")) - // Property("rules", gremlingo.T__.Select("c").Select("rules")) // array of values - return traversal + toStore := utils.ConvertToSliceMapAny(insertsConverted) + log.I.Infof(" ============== INSERTS Role ====== %+v", insertsConverted) + log.I.Infof(" ============== toStore Role ====== %+v", toStore) + g := source.GetGraphTraversal() + for _, i := range toStore { + g = g.AddV(v.Label()). + Property("storeId", i["storeId"]). + Property("name", i["name"]). + Property("is_namespaced", i["is_namespaced"]). + Property("namespace", i["namespace"]) + } + return g + // return g.Inject(toStore).Unfold().As("c"). + // AddV(v.Label()). + // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + // Property("name", gremlingo.T__.Select("c").Select("name")). + // Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). + // Property("namespace", gremlingo.T__.Select("c").Select("namespace")) + // // Property("rules", gremlingo.T__.Select("c").Select("rules")) // array of values } } diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index 10a69694..67f268ef 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -5,7 +5,6 @@ import ( "github.com/DataDog/KubeHound/pkg/telemetry/log" "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" - gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) const ( @@ -26,15 +25,25 @@ func (v Volume) BatchSize() int { } func (v Volume) Traversal() VertexTraversal { - return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { insertsConverted := utils.ConvertSliceAnyToTyped[graph.Volume, TraversalInput](inserts) - log.I.Infof(" ============== INSERTS Volumes ====== %+v", insertsConverted) - traversal := g.Inject(inserts).Unfold().As("c"). - AddV(v.Label()). - Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - Property("name", gremlingo.T__.Select("c").Select("name")). - Property("type", gremlingo.T__.Select("c").Select("type")). - Property("path", gremlingo.T__.Select("c").Select("path")) - return traversal + toStore := utils.ConvertToSliceMapAny(insertsConverted) + log.I.Infof(" ============== INSERTS Volume ====== %+v", insertsConverted) + log.I.Infof(" ============== toStore Volume ====== %+v", toStore) + g := source.GetGraphTraversal() + for _, i := range toStore { + g = g.AddV(v.Label()). + Property("storeId", i["storeId"]). + Property("name", i["name"]). + Property("type", i["type"]). + Property("path", i["path"]) + } + return g + // return g.Inject(inserts).Unfold().As("c"). + // AddV(v.Label()). + // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). + // Property("name", gremlingo.T__.Select("c").Select("name")). + // Property("type", gremlingo.T__.Select("c").Select("type")). + // Property("path", gremlingo.T__.Select("c").Select("path")) } } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 3ea7af3e..285b9664 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -102,6 +102,7 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a op := jgv.gremlin(jgv.traversalSource, convertedToTraversalInput) log.I.Infof("BEFORE ITERATE") promise := op.Iterate() + log.I.Infof("BEFORE AFTER ITERATE") log.I.Infof("BEFORE PROMISE") err := <-promise log.I.Infof("AFTER PROMISE: %v, data: %+v", err, data) diff --git a/pkg/utils/interfaces.go b/pkg/utils/interfaces.go index 1c078968..1e8ef49a 100644 --- a/pkg/utils/interfaces.go +++ b/pkg/utils/interfaces.go @@ -2,6 +2,8 @@ package utils import ( "fmt" + + "github.com/DataDog/KubeHound/pkg/telemetry/log" ) // func AnySliceToStringSlice(in []any) []string { @@ -34,3 +36,15 @@ func ConvertSliceAnyToTyped[T any, Tin any](data []Tin) []T { } return converted } + +func ConvertToSliceMapAny[T any](inserts []T) []map[string]any { + toStore := make([]map[string]any, len(inserts)) + for _, currentStruct := range inserts { + m, err := StructToMap(currentStruct) + if err != nil { + log.I.Errorf("Failed to convert struct to map for Nodes: %+v", err) + } + toStore = append(toStore, m) + } + return toStore +} From 16f2749dffe3f353d8c2eb8571e7b88a74d8a709 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 12:11:34 +0200 Subject: [PATCH 27/61] I WANT TO BELIEVE --- pkg/kubehound/storage/graphdb/janusgraph_writer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 285b9664..16563864 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -105,11 +105,12 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a log.I.Infof("BEFORE AFTER ITERATE") log.I.Infof("BEFORE PROMISE") err := <-promise - log.I.Infof("AFTER PROMISE: %v, data: %+v", err, data) + log.I.Infof("AFTER PROMISE: %v, convertedToTraversalInput: %+v", err, convertedToTraversalInput) if err != nil { jgv.transaction.Rollback() return err } + jgv.transaction.Commit() log.I.Infof("=== DONE == batch write JanusGraphAsyncVertexWriter") return nil } @@ -138,7 +139,7 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any jge.transaction.Rollback() return err } - + jge.transaction.Commit() log.I.Infof("=== DONE == batch write JanusGraphAsyncEdgeWriter") return nil } From f43c018d595e75c9062577e6662ea1a322836008 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 12:23:23 +0200 Subject: [PATCH 28/61] making sure we commit --- pkg/kubehound/storage/graphdb/janusgraph_writer.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 16563864..158fd318 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -110,7 +110,12 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a jgv.transaction.Rollback() return err } - jgv.transaction.Commit() + log.I.Infof("commiting work") + err = jgv.transaction.Commit() + if err != nil { + log.I.Errorf("failed to commit: %+v", err) + return err + } log.I.Infof("=== DONE == batch write JanusGraphAsyncVertexWriter") return nil } @@ -139,7 +144,11 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any jge.transaction.Rollback() return err } - jge.transaction.Commit() + err = jge.transaction.Commit() + if err != nil { + log.I.Errorf("failed to commit: %+v", err) + return err + } log.I.Infof("=== DONE == batch write JanusGraphAsyncEdgeWriter") return nil } From 666946566e45230057f973843675c259f34e6652 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 15:00:24 +0200 Subject: [PATCH 29/61] NODES WORKS PROGRESS --- pkg/kubehound/graph/builder.go | 1 + pkg/kubehound/graph/vertex/identity.go | 5 -- pkg/kubehound/graph/vertex/node.go | 58 ++++--------------- .../ingestor/pipeline/ingest_resources.go | 6 ++ .../storage/graphdb/janusgraph_writer.go | 6 +- 5 files changed, 23 insertions(+), 53 deletions(-) diff --git a/pkg/kubehound/graph/builder.go b/pkg/kubehound/graph/builder.go index f19ec15d..7e915828 100644 --- a/pkg/kubehound/graph/builder.go +++ b/pkg/kubehound/graph/builder.go @@ -47,6 +47,7 @@ func (b *Builder) HealthCheck(ctx context.Context) error { // buildEdge inserts a class of edges into the graph database. // NOTE: function is blocking and expected to be called from within a goroutine. func (b *Builder) buildEdge(ctx context.Context, e edge.Builder) error { + log.I.Infof("entering buildEdge with edge.Builder: %+v", e) w, err := b.graphdb.EdgeWriter(ctx, e) if err != nil { return err diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 8009dd7c..5fcd5e11 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -32,11 +32,6 @@ func (v Identity) Traversal() VertexTraversal { log.I.Infof(" ============== toStore Identity ====== %+v", toStore) g := source.GetGraphTraversal() for _, i := range toStore { - // i := insert.(map[string]any) - // command := utils.ToSliceOfAny(i["command"].([]any)) - // args := utils.ToSliceOfAny(i["args"].([]any)) - // capabilities := utils.ToSliceOfAny(i["capabilities"].([]any)) - g = g.AddV(v.Label()). Property("storeId", i["storeId"]). Property("name", i["name"]). diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 1fe0227b..97a966cf 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -1,9 +1,9 @@ package vertex import ( + "fmt" + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - "github.com/DataDog/KubeHound/pkg/telemetry/log" - "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -26,54 +26,18 @@ func (v Node) BatchSize() int { func (v Node) Traversal() VertexTraversal { return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - insertsConverted := utils.ConvertSliceAnyToTyped[graph.Node, TraversalInput](inserts) - toStore := utils.ConvertToSliceMapAny(insertsConverted) - log.I.Infof(" ============== INSERTS Nodes ====== %+v", insertsConverted) - log.I.Infof(" ============== toStore Nodes ====== %+v", toStore) g := source.GetGraphTraversal() - for _, i := range toStore { - // i := insert.(map[string]any) - // command := utils.ToSliceOfAny(i["command"].([]any)) - // args := utils.ToSliceOfAny(i["args"].([]any)) - // capabilities := utils.ToSliceOfAny(i["capabilities"].([]any)) - + for _, w := range inserts { + data := w.(*graph.Node) + fmt.Printf("WHAT THE FUCK IS THIS INSERTS DATA??????????????????? (cast) %+v\n", data) g = g.AddV(v.Label()). - Property("storeId", i["storeId"]). - Property("name", i["name"]). - Property("image", i["image"]). - Property("privileged", i["privileged"]). - Property("privesc", i["privesc"]). - Property("hostPid", i["hostPid"]). - Property("hostPath", i["hostPath"]). - Property("hostNetwork", i["hostNetwork"]). - Property("runAsUser", i["runAsUser"]). - Property("pod", i["pod"]). - Property("node", i["node"]). - // Property("compromised", i["compromised"]). - Property("critical", i["critical"]) - - // for _, cmd := range command { - // traversal = traversal.Property(gremlingo.Cardinality.Set, "command", cmd) - // } - // for _, arg := range args { - // traversal = traversal.Property(gremlingo.Cardinality.Set, "args", arg) - // } - // for _, cap := range capabilities { - // traversal = traversal.Property(gremlingo.Cardinality.Set, "capabilities", cap) - // } - // for _, port := range ports { - // traversal = traversal.Property(gremlingo.Cardinality.Set, "ports", port) - // } + Property("storeId", data.StoreId). + Property("name", data.Name). + Property("is_namespaced", data.IsNamespaced). + Property("namespace", data.Namespace). + Property("compromised", int(data.Compromised)). + Property("critical", data.Critical) } return g - // return g.Inject(toStore).Unfold().As("c"). - // AddV(v.Label()). - // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - // Property("name", gremlingo.T__.Select("c").Select("name")). - // Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - // Property("namespace", gremlingo.T__.Select("c").Select("namespace")). - // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - // Property("critical", gremlingo.T__.Select("c").Select("critical")) - } } diff --git a/pkg/kubehound/ingestor/pipeline/ingest_resources.go b/pkg/kubehound/ingestor/pipeline/ingest_resources.go index 765f4650..196261b6 100644 --- a/pkg/kubehound/ingestor/pipeline/ingest_resources.go +++ b/pkg/kubehound/ingestor/pipeline/ingest_resources.go @@ -10,6 +10,7 @@ import ( "github.com/DataDog/KubeHound/pkg/kubehound/storage/graphdb" "github.com/DataDog/KubeHound/pkg/kubehound/storage/storedb" "github.com/DataDog/KubeHound/pkg/kubehound/store/collections" + "github.com/DataDog/KubeHound/pkg/telemetry/log" "github.com/hashicorp/go-multierror" ) @@ -85,18 +86,23 @@ func WithStoreWriter[T collections.Collection](c T) IngestResourceOption { // WithStoreWriter initializes a bulk graph writer (and registers a cleanup function) for the provided vertex. // To access the writer use the graphWriter(v vertex.Vertex) function. func WithGraphWriter(v vertex.Builder) IngestResourceOption { + log.I.Infof("--- WithGraphWriter: %+v", v) return func(ctx context.Context, rOpts *resourceOptions, deps *Dependencies) error { + log.I.Infof("--- callback func for WithGraphWriter: %+v", v) w, err := deps.GraphDB.VertexWriter(ctx, v) if err != nil { return err } + log.I.Infof("--- deps.GraphDB.VertexWriter : %+v", v) rOpts.graphWriters[v.Label()] = w rOpts.cleanup = append(rOpts.cleanup, func(ctx context.Context) error { return w.Close(ctx) }) + log.I.Infof("--- append cleanup ok : %+v", v) rOpts.flush = append(rOpts.flush, w.Flush) + log.I.Infof("--- all append done : %+v", v) return nil } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index 158fd318..e75a7564 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -199,9 +199,11 @@ func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { log.I.Infof("Flushing remaining of queue for vertices: %+v", v.inserts) err := v.batchWrite(ctx, v.inserts) if err != nil { + log.I.Errorf("Failed to batch write vertex: %+v", err) v.writingInFligth.Wait() return err } + log.I.Info("Done flushing vertices, clearing the queue") v.inserts = nil return nil @@ -220,12 +222,14 @@ func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { e.writingInFligth.Wait() return nil } - + log.I.Infof("Flushing remaining of queue for edges: %+v", e.inserts) err := e.batchWrite(ctx, e.inserts) if err != nil { + log.I.Errorf("Failed to batch write edge: %+v", err) e.writingInFligth.Wait() return err } + log.I.Info("Done flushing edges, clearing the queue") e.inserts = nil return nil From 957f09eaa5f093fed55ac2bf074cfa813c3a06f1 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 15:37:37 +0200 Subject: [PATCH 30/61] cleanup start --- pkg/kubehound/graph/vertex/container.go | 107 +++++------------- pkg/kubehound/graph/vertex/identity.go | 25 ++-- pkg/kubehound/graph/vertex/node.go | 3 - pkg/kubehound/graph/vertex/pod.go | 34 ++---- pkg/kubehound/graph/vertex/role.go | 24 +--- pkg/kubehound/graph/vertex/volume.go | 23 +--- .../ingestor/pipeline/ingest_resources.go | 7 -- 7 files changed, 59 insertions(+), 164 deletions(-) diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 8992fb8e..42c72adb 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -2,8 +2,6 @@ package vertex import ( "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - "github.com/DataDog/KubeHound/pkg/telemetry/log" - "github.com/DataDog/KubeHound/pkg/utils" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -26,85 +24,38 @@ func (v Container) BatchSize() int { func (v Container) Traversal() VertexTraversal { return func(source *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { - // g = g.GetGraphTraversal() - - // for _, insert := range inserts { - // i := insert.(map[string]any) - // // command := utils.ToSliceOfAny(i["command"].([]any)) - // // args := utils.ToSliceOfAny(i["args"].([]any)) - // // capabilities := utils.ToSliceOfAny(i["capabilities"].([]any)) - - // traversal := g.AddV(v.Label()). - // Property("storeId", i["storeId"]). - // Property("name", i["name"]). - // Property("image", i["image"]). - // Property("privileged", i["privileged"]). - // Property("privesc", i["privesc"]). - // Property("hostPid", i["hostPid"]). - // Property("hostPath", i["hostPath"]). - // Property("hostNetwork", i["hostNetwork"]). - // Property("runAsUser", i["runAsUser"]). - // Property("pod", i["pod"]). - // Property("node", i["node"]). - // Property("compromised", i["compromised"]). - // Property("critical", i["critical"]) - - // // for _, cmd := range command { - // // traversal = traversal.Property(gremlingo.Cardinality.Set, "command", cmd) - // // } - // // for _, arg := range args { - // // traversal = traversal.Property(gremlingo.Cardinality.Set, "args", arg) - // // } - // // for _, cap := range capabilities { - // // traversal = traversal.Property(gremlingo.Cardinality.Set, "capabilities", cap) - // // } - // // for _, port := range ports { - // // traversal = traversal.Property(gremlingo.Cardinality.Set, "ports", port) - // // } - // } - - // return g.GetGraphTraversal() - // I have no idea how I can convert the arrays in the inserts to multiple values with cardinality set.... - insertsConverted := utils.ConvertSliceAnyToTyped[graph.Container, TraversalInput](inserts) - toStore := utils.ConvertToSliceMapAny(insertsConverted) - log.I.Infof(" ============== INSERTS Container ====== %+v", insertsConverted) - log.I.Infof(" ============== toStore Container ====== %+v", toStore) g := source.GetGraphTraversal() - for _, i := range toStore { + + for _, insert := range inserts { + i := insert.(*graph.Container) g = g.AddV(v.Label()). - Property("storeId", i["storeId"]). - Property("name", i["name"]). - Property("image", i["image"]). - Property("privileged", i["privileged"]). - Property("privesc", i["privesc"]). - Property("hostPid", i["hostPid"]). - Property("hostPath", i["hostPath"]). - Property("hostNetwork", i["hostNetwork"]). - Property("runAsUser", i["runAsUser"]). - Property("pod", i["pod"]). - Property("node", i["node"]). - // Property("compromised", i["compromised"]). - Property("critical", i["critical"]) + Property("storeId", i.StoreId). + Property("name", i.Name). + Property("image", i.Image). + Property("privileged", i.Privileged). + Property("privesc", i.PrivEsc). + Property("hostPid", i.HostPID). + Property("hostPath", i.HostPath). + Property("hostNetwork", i.HostNetwork). + Property("runAsUser", i.RunAsUser). + Property("pod", i.Pod). + Property("node", i.Node). + Property("compromised", int(i.Compromised)). + Property("critical", i.Critical) + + for _, cmd := range i.Command { + g = g.Property(gremlingo.Cardinality.Set, "command", cmd) + } + for _, arg := range i.Args { + g = g.Property(gremlingo.Cardinality.Set, "args", arg) + } + for _, cap := range i.Capabilities { + g = g.Property(gremlingo.Cardinality.Set, "capabilities", cap) + } + for _, port := range i.Ports { + g = g.Property(gremlingo.Cardinality.Set, "ports", port) + } } return g - // return g.Inject(toStore).Unfold().As("c"). - // AddV(v.Label()). - // Property("storeId", gremlingo.T__.Select("c").Select("store_id")). - // Property("name", gremlingo.T__.Select("c").Select("name")). - // Property("image", gremlingo.T__.Select("c").Select("image")). - // // Property(gremlingo.Cardinality.Set, "command", gremlingo.T__.Select("c").Select("command")). - // // Property(gremlingo.Cardinality.Set, "args", gremlingo.T__.Select("c").Select("args")). - // // Property(gremlingo.Cardinality.Set, "capabilities", gremlingo.T__.Select("c").Select("capabilities")). - // Property("privileged", gremlingo.T__.Select("c").Select("privileged")). - // Property("privesc", gremlingo.T__.Select("c").Select("privesc")). - // Property("hostPid", gremlingo.T__.Select("c").Select("hostPid")). - // Property("hostPath", gremlingo.T__.Select("c").Select("hostPath")). - // Property("hostNetwork", gremlingo.T__.Select("c").Select("hostNetwork")). - // Property("runAsUser", gremlingo.T__.Select("c").Select("runAsUser")). - // // Property(gremlingo.Cardinality.Set, "ports", gremlingo.T__.Select("c").Select("ports")). - // Property("pod", gremlingo.T__.Select("c").Select("pod")). - // Property("node", gremlingo.T__.Select("c").Select("node")). - // // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - // Property("critical", gremlingo.T__.Select("c").Select("critical")) } } diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 5fcd5e11..932ab2da 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -2,8 +2,6 @@ package vertex import ( "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - "github.com/DataDog/KubeHound/pkg/telemetry/log" - "github.com/DataDog/KubeHound/pkg/utils" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -26,25 +24,16 @@ func (v Identity) BatchSize() int { func (v Identity) Traversal() VertexTraversal { return func(source *gremlingo.GraphTraversalSource, inserts []TraversalInput) *gremlingo.GraphTraversal { - insertsConverted := utils.ConvertSliceAnyToTyped[graph.Identity, TraversalInput](inserts) - toStore := utils.ConvertToSliceMapAny(insertsConverted) - log.I.Infof(" ============== INSERTS Identity ====== %+v", insertsConverted) - log.I.Infof(" ============== toStore Identity ====== %+v", toStore) g := source.GetGraphTraversal() - for _, i := range toStore { + for _, i := range inserts { + data := i.(*graph.Identity) g = g.AddV(v.Label()). - Property("storeId", i["storeId"]). - Property("name", i["name"]). - Property("isNamespaced", i["isNamespaced"]). - Property("namespace", i["namespace"]). - Property("type", i["type"]) + Property("storeId", data.StoreId). + Property("name", data.Name). + Property("isNamespaced", data.IsNamespaced). + Property("namespace", data.Namespace). + Property("type", data.Type) } return g - // return g.Inject(toStore).Unfold().As("c").AddV(v.Label()). - // Property("storeId", gremlingo.T__.Select("c").Select("store_id")). - // Property("name", gremlingo.T__.Select("c").Select("name")). - // Property("isNamespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - // Property("namespace", gremlingo.T__.Select("c").Select("namespace")). - // Property("type", gremlingo.T__.Select("c").Select("type")) } } diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 97a966cf..36556101 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -1,8 +1,6 @@ package vertex import ( - "fmt" - "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -29,7 +27,6 @@ func (v Node) Traversal() VertexTraversal { g := source.GetGraphTraversal() for _, w := range inserts { data := w.(*graph.Node) - fmt.Printf("WHAT THE FUCK IS THIS INSERTS DATA??????????????????? (cast) %+v\n", data) g = g.AddV(v.Label()). Property("storeId", data.StoreId). Property("name", data.Name). diff --git a/pkg/kubehound/graph/vertex/pod.go b/pkg/kubehound/graph/vertex/pod.go index 942d7afc..bdcbfad5 100644 --- a/pkg/kubehound/graph/vertex/pod.go +++ b/pkg/kubehound/graph/vertex/pod.go @@ -2,8 +2,6 @@ package vertex import ( "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - "github.com/DataDog/KubeHound/pkg/telemetry/log" - "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -26,30 +24,20 @@ func (v Pod) BatchSize() int { func (v Pod) Traversal() VertexTraversal { return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - insertsConverted := utils.ConvertSliceAnyToTyped[graph.Pod, TraversalInput](inserts) - toStore := utils.ConvertToSliceMapAny(insertsConverted) - log.I.Infof(" ============== INSERTS Pods ====== %+v", insertsConverted) - log.I.Infof(" ============== toStore Pods ====== %+v", toStore) g := source.GetGraphTraversal() - for _, i := range toStore { + for _, i := range inserts { + data := i.(*graph.Pod) g = g.AddV(v.Label()). - Property("storeId", i["storeId"]). - Property("name", i["name"]). - Property("isNamespaced", i["isNamespaced"]). - Property("namespace", i["namespace"]). - Property("type", i["type"]) + Property("store_id", data.StoreId). + Property("name", data.Name). + Property("is_namespaced", data.IsNamespaced). + Property("namespace", data.Namespace). + Property("sharedProcessNamespace", data.SharedProcessNamespace). + Property("serviceAccount", data.ServiceAccount). + Property("node", data.Node). + Property("compromised", int(data.Compromised)). + Property("critical", data.Critical) } return g - // return g.Inject(toStore).Unfold().As("c"). - // AddV(v.Label()). - // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - // Property("name", gremlingo.T__.Select("c").Select("name")). - // Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - // Property("namespace", gremlingo.T__.Select("c").Select("namespace")). - // Property("sharedProcessNamespace", gremlingo.T__.Select("c").Select("sharedProcessNamespace")). - // Property("serviceAccount", gremlingo.T__.Select("c").Select("serviceAccount")). - // Property("node", gremlingo.T__.Select("c").Select("node")). - // Property("compromised", gremlingo.T__.Select("c").Select("compromised")). - // Property("critical", gremlingo.T__.Select("c").Select("critical")) } } diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index b03c472e..67c9366f 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -2,8 +2,6 @@ package vertex import ( "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - "github.com/DataDog/KubeHound/pkg/telemetry/log" - "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -26,25 +24,15 @@ func (v Role) BatchSize() int { func (v Role) Traversal() VertexTraversal { return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - insertsConverted := utils.ConvertSliceAnyToTyped[graph.Role, TraversalInput](inserts) - toStore := utils.ConvertToSliceMapAny(insertsConverted) - log.I.Infof(" ============== INSERTS Role ====== %+v", insertsConverted) - log.I.Infof(" ============== toStore Role ====== %+v", toStore) g := source.GetGraphTraversal() - for _, i := range toStore { + for _, i := range inserts { + data := i.(*graph.Role) g = g.AddV(v.Label()). - Property("storeId", i["storeId"]). - Property("name", i["name"]). - Property("is_namespaced", i["is_namespaced"]). - Property("namespace", i["namespace"]) + Property("store_id", data.StoreId). + Property("name", data.Name). + Property("is_namespaced", data.IsNamespaced). + Property("namespace", data.Namespace) } return g - // return g.Inject(toStore).Unfold().As("c"). - // AddV(v.Label()). - // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - // Property("name", gremlingo.T__.Select("c").Select("name")). - // Property("is_namespaced", gremlingo.T__.Select("c").Select("is_namespaced")). - // Property("namespace", gremlingo.T__.Select("c").Select("namespace")) - // // Property("rules", gremlingo.T__.Select("c").Select("rules")) // array of values } } diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index 67f268ef..96f66920 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -2,8 +2,6 @@ package vertex import ( "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - "github.com/DataDog/KubeHound/pkg/telemetry/log" - "github.com/DataDog/KubeHound/pkg/utils" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -26,24 +24,15 @@ func (v Volume) BatchSize() int { func (v Volume) Traversal() VertexTraversal { return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - insertsConverted := utils.ConvertSliceAnyToTyped[graph.Volume, TraversalInput](inserts) - toStore := utils.ConvertToSliceMapAny(insertsConverted) - log.I.Infof(" ============== INSERTS Volume ====== %+v", insertsConverted) - log.I.Infof(" ============== toStore Volume ====== %+v", toStore) g := source.GetGraphTraversal() - for _, i := range toStore { + for _, i := range inserts { + data := i.(*graph.Volume) g = g.AddV(v.Label()). - Property("storeId", i["storeId"]). - Property("name", i["name"]). - Property("type", i["type"]). - Property("path", i["path"]) + Property("storeId", data.StoreId). + Property("name", data.Name). + Property("type", data.Type). + Property("path", data.Path) } return g - // return g.Inject(inserts).Unfold().As("c"). - // AddV(v.Label()). - // Property("store_id", gremlingo.T__.Select("c").Select("store_id")). - // Property("name", gremlingo.T__.Select("c").Select("name")). - // Property("type", gremlingo.T__.Select("c").Select("type")). - // Property("path", gremlingo.T__.Select("c").Select("path")) } } diff --git a/pkg/kubehound/ingestor/pipeline/ingest_resources.go b/pkg/kubehound/ingestor/pipeline/ingest_resources.go index 196261b6..d3b0714f 100644 --- a/pkg/kubehound/ingestor/pipeline/ingest_resources.go +++ b/pkg/kubehound/ingestor/pipeline/ingest_resources.go @@ -10,7 +10,6 @@ import ( "github.com/DataDog/KubeHound/pkg/kubehound/storage/graphdb" "github.com/DataDog/KubeHound/pkg/kubehound/storage/storedb" "github.com/DataDog/KubeHound/pkg/kubehound/store/collections" - "github.com/DataDog/KubeHound/pkg/telemetry/log" "github.com/hashicorp/go-multierror" ) @@ -86,24 +85,18 @@ func WithStoreWriter[T collections.Collection](c T) IngestResourceOption { // WithStoreWriter initializes a bulk graph writer (and registers a cleanup function) for the provided vertex. // To access the writer use the graphWriter(v vertex.Vertex) function. func WithGraphWriter(v vertex.Builder) IngestResourceOption { - log.I.Infof("--- WithGraphWriter: %+v", v) return func(ctx context.Context, rOpts *resourceOptions, deps *Dependencies) error { - log.I.Infof("--- callback func for WithGraphWriter: %+v", v) w, err := deps.GraphDB.VertexWriter(ctx, v) if err != nil { return err } - log.I.Infof("--- deps.GraphDB.VertexWriter : %+v", v) rOpts.graphWriters[v.Label()] = w rOpts.cleanup = append(rOpts.cleanup, func(ctx context.Context) error { return w.Close(ctx) }) - log.I.Infof("--- append cleanup ok : %+v", v) rOpts.flush = append(rOpts.flush, w.Flush) - log.I.Infof("--- all append done : %+v", v) - return nil } } From 1519bfdc48d8ba8377dedc5fca231461ef811bad Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 17:10:47 +0200 Subject: [PATCH 31/61] ????? --- .../storage/graphdb/janusgraph_writer.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index e75a7564..ed64033e 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -24,6 +24,7 @@ type JanusGraphAsyncVertexWriter struct { consumerChan chan []any writingInFligth sync.WaitGroup batchSize int // Shouldn't this be "per vertex types" ? + mu sync.Mutex } var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) @@ -36,6 +37,7 @@ type JanusGraphAsyncEdgeWriter struct { consumerChan chan []any writingInFligth sync.WaitGroup batchSize int // Shouldn't this be "per edge types" ? + mu sync.Mutex } func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, e edge.Builder, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { @@ -236,21 +238,31 @@ func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { } func (vw *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { + vw.mu.Lock() + defer vw.mu.Unlock() if len(vw.inserts) > vw.batchSize { - vw.consumerChan <- vw.inserts + copied := make([]any, len(vw.inserts)) + copy(copied, vw.inserts) + vw.consumerChan <- copied // cleanup the ops array after we have copied it to the channel vw.inserts = nil } vw.inserts = append(vw.inserts, vertex) + log.I.Errorf("INSERTS AFTER APPEND: %+v", &vw.inserts) return nil } func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { + e.mu.Lock() + defer e.mu.Unlock() if len(e.inserts) > e.batchSize { - e.consumerChan <- e.inserts + copied := make([]any, len(e.inserts)) + copy(copied, e.inserts) + e.consumerChan <- copied // cleanup the ops array after we have copied it to the channel e.inserts = nil } e.inserts = append(e.inserts, edge) + log.I.Errorf("INSERTS AFTER APPEND: %+v", &e.inserts) return nil } From 9ac80270eefff72f36ad873c0491491e38bbc756 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 17:25:54 +0200 Subject: [PATCH 32/61] bla --- pkg/kubehound/ingestor/pipeline/pod_ingest.go | 4 +++- pkg/kubehound/ingestor/pipeline_ingestor.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/ingestor/pipeline/pod_ingest.go b/pkg/kubehound/ingestor/pipeline/pod_ingest.go index 73452155..eb1dcc68 100644 --- a/pkg/kubehound/ingestor/pipeline/pod_ingest.go +++ b/pkg/kubehound/ingestor/pipeline/pod_ingest.go @@ -8,6 +8,7 @@ import ( "github.com/DataDog/KubeHound/pkg/kubehound/models/store" "github.com/DataDog/KubeHound/pkg/kubehound/storage/cache" "github.com/DataDog/KubeHound/pkg/kubehound/store/collections" + "github.com/DataDog/KubeHound/pkg/telemetry/log" ) const ( @@ -108,7 +109,8 @@ func (i *PodIngest) processVolume(ctx context.Context, parent *store.Pod, volume // Normalize volume to store object format sv, err := i.r.storeConvert.Volume(ctx, volume, parent) if err != nil { - return err + log.I.Errorf("CHANGEME: failed to process volume type: %v", err) + return nil } // Async write to store diff --git a/pkg/kubehound/ingestor/pipeline_ingestor.go b/pkg/kubehound/ingestor/pipeline_ingestor.go index 3e9a1d09..ec9cb84d 100644 --- a/pkg/kubehound/ingestor/pipeline_ingestor.go +++ b/pkg/kubehound/ingestor/pipeline_ingestor.go @@ -128,7 +128,7 @@ func (i PipelineIngestor) Run(outer context.Context) error { err := s.Run(ctx, deps) if err != nil { l.Errorf("ingestor sequence %s run: %v", s.Name, err) - cancelAll(err) + // cancelAll(err) } }() } From 6514eb375999776832989fdb72f77ded27a36434 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 17:40:11 +0200 Subject: [PATCH 33/61] remove absolute path --- test/system/kubehound.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/system/kubehound.yaml b/test/system/kubehound.yaml index 793ef172..11b6f9d8 100644 --- a/test/system/kubehound.yaml +++ b/test/system/kubehound.yaml @@ -1,7 +1,7 @@ collector: type: file-collector file: - directory: /home/edouard/dd/KubeHound/test/system/kind-collect/kind-kubehound.test.local + directory: kind-collect/kind-kubehound.test.local janusgraph: url: "ws://localhost:8182/gremlin" mongodb: From 8f18a5b6c364fac5dff6da6ae67517d8ec2338ce Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 17:57:15 +0200 Subject: [PATCH 34/61] add large chan size --- pkg/kubehound/storage/graphdb/janusgraph_writer.go | 4 ++-- pkg/kubehound/storage/storedb/mongo_writer.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index ed64033e..cf323cc6 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -59,7 +59,7 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, e edge. transaction: tx, traversalSource: gtx, batchSize: 1, - consumerChan: make(chan []any, 10), + consumerChan: make(chan []any, 10000), } return &jw, nil @@ -84,7 +84,7 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, v ver transaction: tx, traversalSource: gtx, batchSize: 1, - consumerChan: make(chan []any, 10), + consumerChan: make(chan []any, 10000), } return &jw, nil diff --git a/pkg/kubehound/storage/storedb/mongo_writer.go b/pkg/kubehound/storage/storedb/mongo_writer.go index 5eb218a0..c52aae37 100644 --- a/pkg/kubehound/storage/storedb/mongo_writer.go +++ b/pkg/kubehound/storage/storedb/mongo_writer.go @@ -14,7 +14,7 @@ import ( const ( // TODO: this might need to be adjusted in the future, potentially per type of collections // We don't have the data yet, so lets just hardcode a small-ish value for now - consumerChanSize = 10 + consumerChanSize = 10000 ) var _ AsyncWriter = (*MongoAsyncWriter)(nil) From 159e025ece110d0e6c75dd3269ca5f659bb1011e Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 18:02:56 +0200 Subject: [PATCH 35/61] lower batch size --- pkg/kubehound/graph/vertex/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubehound/graph/vertex/builder.go b/pkg/kubehound/graph/vertex/builder.go index 4ad8f9fc..f4f70701 100644 --- a/pkg/kubehound/graph/vertex/builder.go +++ b/pkg/kubehound/graph/vertex/builder.go @@ -5,7 +5,7 @@ import ( ) const ( - DefaultBatchSize = 100 + DefaultBatchSize = 5 ) // An object to be consumed by a vertex traversal function to insert a vertex into the graph database. From 680b5396c8ff01a9a057c4cfd1beba3a890449c1 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Fri, 9 Jun 2023 18:18:37 +0200 Subject: [PATCH 36/61] Comment out all janus graph transaction --- .../storage/graphdb/janusgraph_writer.go | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_writer.go index cf323cc6..2d735c02 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_writer.go @@ -48,16 +48,16 @@ func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, e edge. } traversal := gremlingo.Traversal_().WithRemote(drc) - tx := traversal.Tx() - gtx, err := tx.Begin() - if err != nil { - return nil, err - } + // tx := traversal.Tx() + // gtx, err := tx.Begin() + // if err != nil { + // return nil, err + // } jw := JanusGraphAsyncEdgeWriter{ - gremlin: e.Traversal(), - inserts: make([]any, 0), - transaction: tx, - traversalSource: gtx, + gremlin: e.Traversal(), + inserts: make([]any, 0), + // transaction: tx, + traversalSource: traversal, batchSize: 1, consumerChan: make(chan []any, 10000), } @@ -73,16 +73,16 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, v ver } source := gremlingo.Traversal_().WithRemote(drc) - tx := source.Tx() - gtx, err := tx.Begin() - if err != nil { - return nil, err - } + // tx := source.Tx() + // gtx, err := tx.Begin() + // if err != nil { + // return nil, err + // } jw := JanusGraphAsyncVertexWriter{ - gremlin: v.Traversal(), - inserts: make([]interface{}, 0), - transaction: tx, - traversalSource: gtx, + gremlin: v.Traversal(), + inserts: make([]interface{}, 0), + // transaction: tx, + traversalSource: source, batchSize: 1, consumerChan: make(chan []any, 10000), } @@ -109,15 +109,15 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a err := <-promise log.I.Infof("AFTER PROMISE: %v, convertedToTraversalInput: %+v", err, convertedToTraversalInput) if err != nil { - jgv.transaction.Rollback() - return err - } - log.I.Infof("commiting work") - err = jgv.transaction.Commit() - if err != nil { - log.I.Errorf("failed to commit: %+v", err) + // jgv.transaction.Rollback() return err } + // log.I.Infof("commiting work") + // err = jgv.transaction.Commit() + // if err != nil { + // log.I.Errorf("failed to commit: %+v", err) + // return err + // } log.I.Infof("=== DONE == batch write JanusGraphAsyncVertexWriter") return nil } @@ -143,14 +143,14 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any promise := op.Iterate() err := <-promise if err != nil { - jge.transaction.Rollback() - return err - } - err = jge.transaction.Commit() - if err != nil { - log.I.Errorf("failed to commit: %+v", err) + // jge.transaction.Rollback() return err } + // err = jge.transaction.Commit() + // if err != nil { + // log.I.Errorf("failed to commit: %+v", err) + // return err + // } log.I.Infof("=== DONE == batch write JanusGraphAsyncEdgeWriter") return nil } @@ -178,11 +178,13 @@ func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { } func (v *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { - return v.transaction.Close() + // return v.transaction.Close() + return nil } func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { - return e.transaction.Close() + // return e.transaction.Close() + return nil } func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { From 2c067060642d99fa38daecf8e076c6ccfdcd4216 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 12:17:44 +0200 Subject: [PATCH 37/61] IT WORKS (I think) --- Makefile | 2 +- pkg/kubehound/core/core.go | 1 + pkg/kubehound/graph/edge/container_attach.go | 34 +++- pkg/kubehound/graph/vertex/container.go | 6 + pkg/kubehound/graph/vertex/container_test.go | 9 +- pkg/kubehound/graph/vertex/node.go | 3 + .../ingestor/pipeline/node_ingest.go | 3 + pkg/kubehound/ingestor/pipeline_ingestor.go | 1 + .../storage/graphdb/janusgraph_edge_writer.go | 149 +++++++++++++++ .../storage/graphdb/janusgraph_provider.go | 4 +- ..._writer.go => janusgraph_vertex_writer.go} | 174 +++--------------- pkg/kubehound/storage/storedb/mongo_writer.go | 2 +- test/system/setup_test.go | 11 +- 13 files changed, 238 insertions(+), 161 deletions(-) create mode 100644 pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go rename pkg/kubehound/storage/graphdb/{janusgraph_writer.go => janusgraph_vertex_writer.go} (51%) diff --git a/Makefile b/Makefile index 4a09c8ea..78935853 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test: .PHONY: system-test system-test: - cd test/system && go test -v -timeout "30s" ./... + cd test/system && go test -v -timeout "30s" -race ./... .PHONY: local-cluster-create local-cluster-setup: diff --git a/pkg/kubehound/core/core.go b/pkg/kubehound/core/core.go index ac510e1d..95d7dbd1 100644 --- a/pkg/kubehound/core/core.go +++ b/pkg/kubehound/core/core.go @@ -127,6 +127,7 @@ func Launch(ctx context.Context, opts ...LaunchOption) error { log.I.Info("Building attack graph") if err := buildGraph(ctx, cfg, sp, gp); err != nil { + log.I.Errorf("!!!!!!!! FAILING TO BUILD GRAPH: %v!!!!!!!!", err) return fmt.Errorf("building attack graph: %w", err) } diff --git a/pkg/kubehound/graph/edge/container_attach.go b/pkg/kubehound/graph/edge/container_attach.go index 79fbf973..30677aa8 100644 --- a/pkg/kubehound/graph/edge/container_attach.go +++ b/pkg/kubehound/graph/edge/container_attach.go @@ -33,12 +33,34 @@ func (e ContainerAttach) BatchSize() int { } func (e ContainerAttach) Traversal() EdgeTraversal { - return func(g *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - log.I.Errorf("CONVERT ME TO SOMETHING TYPED OTHERWISE THIS WILL BROKE") - return g.Inject(inserts).Unfold().As("ca"). - V().HasLabel("Pod").Has("storeId", gremlin.T__.Select("ca").Select("pod")).As("pod"). - V().HasLabel("Container").Has("storeId", gremlin.T__.Select("ca").Select("container")).As("container"). - MergeE(e.Label()).From("pod").To("container") + return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { + log.I.Errorf("CONVERT ME TO SOMETHING TYPED OTHERWISE THIS WILL BREAK") + + g := source.GetGraphTraversal() + g.AddV("dummy_entry") + // for _, i := range inserts { + // data := i.(*ContainerAttachGroup) + // podId, err := data.Pod.MarshalJSON() + // if err != nil { + // log.I.Errorf("failed to get pod id: %v", err) + // } + + // for _, container := range data.Containers { + // containerID, err := container.MarshalJSON() + // if err != nil { + // log.I.Errorf("failed to get pod id: %v", err) + // } + // pod, err := g.V("Pod").Has("store_id", podId).Next() + // containers, err := g.V("Container").Has("store_id", containerID).Next() + // clist, err := containers.GetSlice() + + // for _, container := range *clist { + // fmt.Printf("containers edge blabla %+v\n", container) + // g = g.V(pod).AddE(e.Label()).To(container) + // } + // } + // } + return g } } diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 42c72adb..923191cb 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,6 +1,8 @@ package vertex import ( + "fmt" + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -43,15 +45,19 @@ func (v Container) Traversal() VertexTraversal { Property("compromised", int(i.Compromised)). Property("critical", i.Critical) + fmt.Printf("@@@@@@@@ Command for container: %+v\n", i.Command) for _, cmd := range i.Command { g = g.Property(gremlingo.Cardinality.Set, "command", cmd) } + fmt.Printf("@@@@@@@@ Args for container: %+v\n", i.Args) for _, arg := range i.Args { g = g.Property(gremlingo.Cardinality.Set, "args", arg) } + fmt.Printf("@@@@@@@@ Capabilities for container: %+v\n", i.Capabilities) for _, cap := range i.Capabilities { g = g.Property(gremlingo.Cardinality.Set, "capabilities", cap) } + fmt.Printf("@@@@@@@@ Ports for container: %+v\n", i.Ports) for _, port := range i.Ports { g = g.Property(gremlingo.Cardinality.Set, "ports", port) } diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go index ab684ce9..dc6a8656 100644 --- a/pkg/kubehound/graph/vertex/container_test.go +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" - "github.com/DataDog/KubeHound/pkg/utils" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" "github.com/stretchr/testify/assert" ) @@ -27,7 +26,7 @@ func TestContainer_Traversal(t *testing.T) { Name: "test name", Image: "image", Command: []string{"/usr/bin/sleep"}, - Args: []string{"600"}, + Args: []string{"600", "lol2ndarguments"}, Capabilities: []string{"NET_CAP_ADMIN", "NET_RAW_ADMIN"}, Privileged: true, PrivEsc: true, @@ -51,11 +50,9 @@ func TestContainer_Traversal(t *testing.T) { v := Container{} g := gremlingo.GraphTraversalSource{} - insert, err := utils.StructToMap(tt.data) - assert.NoError(t, err) vertexTraversal := v.Traversal() - inserts := []TraversalInput{insert} + inserts := []TraversalInput{&tt.data} fmt.Printf("inserts: %v\n", inserts) traversal := vertexTraversal(&g, inserts) @@ -64,6 +61,8 @@ func TestContainer_Traversal(t *testing.T) { assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "/usr/bin/sleep") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "600") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "lol2ndarguments") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "1337") }) } diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 36556101..519f1bb1 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -1,6 +1,8 @@ package vertex import ( + "fmt" + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -27,6 +29,7 @@ func (v Node) Traversal() VertexTraversal { g := source.GetGraphTraversal() for _, w := range inserts { data := w.(*graph.Node) + fmt.Printf("QWERTY: %v", data) g = g.AddV(v.Label()). Property("storeId", data.StoreId). Property("name", data.Name). diff --git a/pkg/kubehound/ingestor/pipeline/node_ingest.go b/pkg/kubehound/ingestor/pipeline/node_ingest.go index 34e7630a..2fc3fd75 100644 --- a/pkg/kubehound/ingestor/pipeline/node_ingest.go +++ b/pkg/kubehound/ingestor/pipeline/node_ingest.go @@ -7,6 +7,7 @@ import ( "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" "github.com/DataDog/KubeHound/pkg/kubehound/storage/cache" "github.com/DataDog/KubeHound/pkg/kubehound/store/collections" + "github.com/DataDog/KubeHound/pkg/telemetry/log" ) const ( @@ -45,6 +46,7 @@ func (i *NodeIngest) Initialize(ctx context.Context, deps *Dependencies) error { // streamCallback is invoked by the collector for each node collected. // The function ingests an input node into the cache/store/graph databases asynchronously. func (i *NodeIngest) IngestNode(ctx context.Context, node types.NodeType) error { + log.I.Infof("+++++++ INGEST NODE: (%s) ", node.Name) // Normalize node to store object format o, err := i.r.storeConvert.Node(ctx, node) if err != nil { @@ -82,6 +84,7 @@ func (i *NodeIngest) Complete(ctx context.Context) error { } func (i *NodeIngest) Run(ctx context.Context) error { + log.I.Infof("!!!!!! run NodeIngest !!!!!!!") return i.r.collect.StreamNodes(ctx, i) } diff --git a/pkg/kubehound/ingestor/pipeline_ingestor.go b/pkg/kubehound/ingestor/pipeline_ingestor.go index ec9cb84d..dc3d97e6 100644 --- a/pkg/kubehound/ingestor/pipeline_ingestor.go +++ b/pkg/kubehound/ingestor/pipeline_ingestor.go @@ -127,6 +127,7 @@ func (i PipelineIngestor) Run(outer context.Context) error { err := s.Run(ctx, deps) if err != nil { + log.I.Errorf("!!!!!!!!!! failed to run pipeline sequence thing !!!!!!!!: %v", err) l.Errorf("ingestor sequence %s run: %v", s.Name, err) // cancelAll(err) } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go new file mode 100644 index 00000000..be1b9004 --- /dev/null +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -0,0 +1,149 @@ +package graphdb + +import ( + "context" + "errors" + "sync" + + "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" + "github.com/DataDog/KubeHound/pkg/telemetry/log" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" +) + +var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) + +type JanusGraphAsyncEdgeWriter struct { + gremlin edge.EdgeTraversal + transaction *gremlingo.Transaction + traversalSource *gremlingo.GraphTraversalSource + inserts []any + consumerChan chan []any + writingInFligth sync.WaitGroup + batchSize int // Shouldn't this be "per edge types" ? + mu sync.Mutex +} + +func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, e edge.Builder, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { + log.I.Infof("Created new JanusGraphAsyncEdgeWriter") + options := &writerOptions{} + for _, opt := range opts { + opt(options) + } + + traversal := gremlingo.Traversal_().WithRemote(drc) + // tx := traversal.Tx() + // gtx, err := tx.Begin() + // if err != nil { + // return nil, err + // } + jw := JanusGraphAsyncEdgeWriter{ + gremlin: e.Traversal(), + inserts: make([]any, 0), + // transaction: tx, + traversalSource: traversal, + batchSize: 1, + consumerChan: make(chan []any, 10000), + } + jw.backgroundWriter(ctx) + return &jw, nil +} + +// backgroundWriter starts a background go routine +func (jge *JanusGraphAsyncEdgeWriter) backgroundWriter(ctx context.Context) { + go func() { + for { + select { + case data := <-jge.consumerChan: + // closing the channel shoud stop the go routine + if data == nil { + return + } + err := jge.batchWrite(ctx, data) + if err != nil { + log.I.Errorf("failed to write data in background batch writer: %v", err) + } + case <-ctx.Done(): + log.I.Info("Closed background mongodb worker") + return + } + } + }() +} + +func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any) error { + log.I.Infof("batch write JanusGraphAsyncEdgeWriter") + jge.writingInFligth.Add(1) + defer jge.writingInFligth.Done() + + // This seems ~pointless BUT is required to have the ability to use edge.TraversalInput/vertex.TraversalInput + // as the type + // Even tho it's an alias to any, since we use it in an array, we cannot simply .([]any) or vice versa because of the underlying memory layout. + convertedToTraversalInput := make([]edge.TraversalInput, 0) + for _, d := range data { + convertedToTraversalInput = append(convertedToTraversalInput, d.(edge.TraversalInput)) + } + + op := jge.gremlin(jge.traversalSource, convertedToTraversalInput) + promise := op.Iterate() + err := <-promise + if err != nil { + // jge.transaction.Rollback() + return err + } + // err = jge.transaction.Commit() + // if err != nil { + // log.I.Errorf("failed to commit: %+v", err) + // return err + // } + log.I.Infof("=== DONE == batch write JanusGraphAsyncEdgeWriter") + return nil +} + +func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { + // return e.transaction.Close() + return nil +} + +func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + + if e.traversalSource == nil { + return errors.New("JanusGraph traversalSource is not initialized") + } + + if len(e.inserts) == 0 { + log.I.Debugf("Skipping flush on edges writer as no write operations left") + // we need to send something to the channel from this function whenever we don't return an error + // we cannot defer it because the go routine may last longer than the current function + // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data + e.writingInFligth.Wait() + return nil + } + log.I.Infof("Flushing remaining of queue for edges: %+v", e.inserts) + err := e.batchWrite(ctx, e.inserts) + if err != nil { + log.I.Errorf("Failed to batch write edge: %+v", err) + e.writingInFligth.Wait() + return err + } + log.I.Info("Done flushing edges, clearing the queue") + e.inserts = nil + + return nil +} + +func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { + e.mu.Lock() + defer e.mu.Unlock() + if len(e.inserts) > e.batchSize { + copied := make([]any, len(e.inserts)) + copy(copied, e.inserts) + e.consumerChan <- copied + // cleanup the ops array after we have copied it to the channel + e.inserts = nil + } + e.inserts = append(e.inserts, edge) + log.I.Errorf("INSERTS AFTER APPEND (edge): %+v", &e.inserts) + return nil +} diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 975bd8fe..11450da5 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -49,7 +49,7 @@ func (jgp *JanusGraphProvider) Raw() any { // VertexWriter creates a new AsyncVertexWriter instance to enable asynchronous bulk inserts of vertices. func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Builder, opts ...WriterOption) (AsyncVertexWriter, error) { - writer, err := NewJanusGraphAsyncVertexWriter(jgp.client, v, opts...) + writer, err := NewJanusGraphAsyncVertexWriter(ctx, jgp.client, v, opts...) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Builde // EdgeWriter creates a new AsyncEdgeWriter instance to enable asynchronous bulk inserts of edges. func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Builder, opts ...WriterOption) (AsyncEdgeWriter, error) { - writer, err := NewJanusGraphAsyncEdgeWriter(jgp.client, e) + writer, err := NewJanusGraphAsyncEdgeWriter(ctx, jgp.client, e) if err != nil { return nil, err } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go similarity index 51% rename from pkg/kubehound/storage/graphdb/janusgraph_writer.go rename to pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index 2d735c02..728bc4b4 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -5,7 +5,6 @@ import ( "errors" "sync" - "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" "github.com/DataDog/KubeHound/pkg/telemetry/log" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" @@ -27,45 +26,7 @@ type JanusGraphAsyncVertexWriter struct { mu sync.Mutex } -var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) - -type JanusGraphAsyncEdgeWriter struct { - gremlin edge.EdgeTraversal - transaction *gremlingo.Transaction - traversalSource *gremlingo.GraphTraversalSource - inserts []any - consumerChan chan []any - writingInFligth sync.WaitGroup - batchSize int // Shouldn't this be "per edge types" ? - mu sync.Mutex -} - -func NewJanusGraphAsyncEdgeWriter(drc *gremlingo.DriverRemoteConnection, e edge.Builder, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { - log.I.Infof("Created new JanusGraphAsyncEdgeWriter") - options := &writerOptions{} - for _, opt := range opts { - opt(options) - } - - traversal := gremlingo.Traversal_().WithRemote(drc) - // tx := traversal.Tx() - // gtx, err := tx.Begin() - // if err != nil { - // return nil, err - // } - jw := JanusGraphAsyncEdgeWriter{ - gremlin: e.Traversal(), - inserts: make([]any, 0), - // transaction: tx, - traversalSource: traversal, - batchSize: 1, - consumerChan: make(chan []any, 10000), - } - - return &jw, nil -} - -func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, v vertex.Builder, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { +func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, v vertex.Builder, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { log.I.Infof("Created new JanusGraphAsyncVertexWriter") options := &writerOptions{} for _, opt := range opts { @@ -86,10 +47,32 @@ func NewJanusGraphAsyncVertexWriter(drc *gremlingo.DriverRemoteConnection, v ver batchSize: 1, consumerChan: make(chan []any, 10000), } - + jw.backgroundWriter(ctx) return &jw, nil } +// backgroundWriter starts a background go routine +func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { + go func() { + for { + select { + case data := <-jgv.consumerChan: + // closing the channel shoud stop the go routine + if data == nil { + return + } + err := jgv.batchWrite(ctx, data) + if err != nil { + log.I.Errorf("failed to write data in background batch writer: %v", err) + } + case <-ctx.Done(): + log.I.Info("Closed background mongodb worker") + return + } + } + }() +} + func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []any) error { log.I.Infof("batch write JanusGraphAsyncVertexWriter") jgv.writingInFligth.Add(1) @@ -100,15 +83,15 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a convertedToTraversalInput = append(convertedToTraversalInput, d.(vertex.TraversalInput)) } - log.I.Infof("BEFORE gremlin()") + log.I.Infof("BEFORE gremlin(), traversal input: %+v", &convertedToTraversalInput) op := jgv.gremlin(jgv.traversalSource, convertedToTraversalInput) log.I.Infof("BEFORE ITERATE") promise := op.Iterate() - log.I.Infof("BEFORE AFTER ITERATE") log.I.Infof("BEFORE PROMISE") err := <-promise log.I.Infof("AFTER PROMISE: %v, convertedToTraversalInput: %+v", err, convertedToTraversalInput) if err != nil { + log.I.Infof("IS THERE AN ERROR HERE: %+v", err) // jgv.transaction.Rollback() return err } @@ -122,72 +105,15 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a return nil } -func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any) error { - log.I.Infof("batch write JanusGraphAsyncEdgeWriter") - jge.writingInFligth.Add(1) - defer jge.writingInFligth.Done() - - if jge.gremlin == nil { - panic("lol") - } - - // This seems ~pointless BUT is required to have the ability to use edge.TraversalInput/vertex.TraversalInput - // as the type - // Even tho it's an alias to any, since we use it in an array, we cannot simply .([]any) or vice versa because of the underlying memory layout. - convertedToTraversalInput := make([]edge.TraversalInput, 0) - for _, d := range data { - convertedToTraversalInput = append(convertedToTraversalInput, d.(edge.TraversalInput)) - } - - op := jge.gremlin(jge.traversalSource, convertedToTraversalInput) - promise := op.Iterate() - err := <-promise - if err != nil { - // jge.transaction.Rollback() - return err - } - // err = jge.transaction.Commit() - // if err != nil { - // log.I.Errorf("failed to commit: %+v", err) - // return err - // } - log.I.Infof("=== DONE == batch write JanusGraphAsyncEdgeWriter") - return nil -} - -// backgroundWriter starts a background go routine -func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { - go func() { - for { - select { - case data := <-jgv.consumerChan: - // closing the channel shoud stop the go routine - if data == nil { - return - } - err := jgv.batchWrite(ctx, data) - if err != nil { - log.I.Errorf("failed to write data in background batch writer: %v", err) - } - case <-ctx.Done(): - log.I.Info("Closed background Janus Graph worker (vertex)") - return - } - } - }() -} - func (v *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { // return v.transaction.Close() return nil } -func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { - // return e.transaction.Close() - return nil -} - func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { + v.mu.Lock() + defer v.mu.Unlock() + if v.traversalSource == nil { return errors.New("JanusGraph traversalSource is not initialized") } @@ -213,32 +139,6 @@ func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { return nil } -func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { - if e.traversalSource == nil { - return errors.New("JanusGraph traversalSource is not initialized") - } - - if len(e.inserts) == 0 { - log.I.Debugf("Skipping flush on edges writer as no write operations left") - // we need to send something to the channel from this function whenever we don't return an error - // we cannot defer it because the go routine may last longer than the current function - // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data - e.writingInFligth.Wait() - return nil - } - log.I.Infof("Flushing remaining of queue for edges: %+v", e.inserts) - err := e.batchWrite(ctx, e.inserts) - if err != nil { - log.I.Errorf("Failed to batch write edge: %+v", err) - e.writingInFligth.Wait() - return err - } - log.I.Info("Done flushing edges, clearing the queue") - e.inserts = nil - - return nil -} - func (vw *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { vw.mu.Lock() defer vw.mu.Unlock() @@ -250,21 +150,5 @@ func (vw *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) er vw.inserts = nil } vw.inserts = append(vw.inserts, vertex) - log.I.Errorf("INSERTS AFTER APPEND: %+v", &vw.inserts) - return nil -} - -func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { - e.mu.Lock() - defer e.mu.Unlock() - if len(e.inserts) > e.batchSize { - copied := make([]any, len(e.inserts)) - copy(copied, e.inserts) - e.consumerChan <- copied - // cleanup the ops array after we have copied it to the channel - e.inserts = nil - } - e.inserts = append(e.inserts, edge) - log.I.Errorf("INSERTS AFTER APPEND: %+v", &e.inserts) return nil } diff --git a/pkg/kubehound/storage/storedb/mongo_writer.go b/pkg/kubehound/storage/storedb/mongo_writer.go index c52aae37..0e06ddfc 100644 --- a/pkg/kubehound/storage/storedb/mongo_writer.go +++ b/pkg/kubehound/storage/storedb/mongo_writer.go @@ -51,7 +51,7 @@ func (maw *MongoAsyncWriter) backgroundWriter(ctx context.Context) { } err := maw.batchWrite(ctx, data) if err != nil { - log.I.Errorf("failed to write data in background batch writer: %w", err) + log.I.Errorf("failed to write data in background batch writer: %v", err) } case <-ctx.Done(): log.I.Info("Closed background mongodb worker") diff --git a/test/system/setup_test.go b/test/system/setup_test.go index a2041564..6dc2424b 100644 --- a/test/system/setup_test.go +++ b/test/system/setup_test.go @@ -41,7 +41,7 @@ func runKubeHound() error { cmd := exec.CommandContext(cmdCtx, CollectorScriptPath, CollectorOutputDir) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - defer cleanupCollected() + // defer cleanupCollected() if err := cmd.Run(); err != nil { return fmt.Errorf("collector script execution: %v", err) } @@ -63,3 +63,12 @@ func TestMain(m *testing.M) { // Run the test suite os.Exit(m.Run()) } + +// func TestDebugE(m *testing.T) { +// if err := runKubeHound(); err != nil { +// log.I.Fatalf(err.Error()) +// } + +// // Run the test suite +// // os.Exit(m.Run()) +// } From e66cb028c29458305c78d701679da363667d0a73 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 12:41:01 +0200 Subject: [PATCH 38/61] cleanup 1 --- pkg/kubehound/core/core.go | 1 - pkg/kubehound/graph/vertex/container.go | 6 -- pkg/kubehound/graph/vertex/container_test.go | 1 - pkg/kubehound/graph/vertex/node.go | 3 - .../ingestor/pipeline/node_ingest.go | 3 - pkg/kubehound/ingestor/pipeline_ingestor.go | 1 - .../storage/graphdb/janusgraph_edge_writer.go | 71 ++++++++++-------- .../graphdb/janusgraph_vertex_writer.go | 73 ++++++++++--------- pkg/kubehound/storage/graphdb/provider.go | 1 + test/system/setup_test.go | 2 +- 10 files changed, 82 insertions(+), 80 deletions(-) diff --git a/pkg/kubehound/core/core.go b/pkg/kubehound/core/core.go index 95d7dbd1..ac510e1d 100644 --- a/pkg/kubehound/core/core.go +++ b/pkg/kubehound/core/core.go @@ -127,7 +127,6 @@ func Launch(ctx context.Context, opts ...LaunchOption) error { log.I.Info("Building attack graph") if err := buildGraph(ctx, cfg, sp, gp); err != nil { - log.I.Errorf("!!!!!!!! FAILING TO BUILD GRAPH: %v!!!!!!!!", err) return fmt.Errorf("building attack graph: %w", err) } diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 923191cb..42c72adb 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -1,8 +1,6 @@ package vertex import ( - "fmt" - "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -45,19 +43,15 @@ func (v Container) Traversal() VertexTraversal { Property("compromised", int(i.Compromised)). Property("critical", i.Critical) - fmt.Printf("@@@@@@@@ Command for container: %+v\n", i.Command) for _, cmd := range i.Command { g = g.Property(gremlingo.Cardinality.Set, "command", cmd) } - fmt.Printf("@@@@@@@@ Args for container: %+v\n", i.Args) for _, arg := range i.Args { g = g.Property(gremlingo.Cardinality.Set, "args", arg) } - fmt.Printf("@@@@@@@@ Capabilities for container: %+v\n", i.Capabilities) for _, cap := range i.Capabilities { g = g.Property(gremlingo.Cardinality.Set, "capabilities", cap) } - fmt.Printf("@@@@@@@@ Ports for container: %+v\n", i.Ports) for _, port := range i.Ports { g = g.Property(gremlingo.Cardinality.Set, "ports", port) } diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go index dc6a8656..5c8dc162 100644 --- a/pkg/kubehound/graph/vertex/container_test.go +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -54,7 +54,6 @@ func TestContainer_Traversal(t *testing.T) { vertexTraversal := v.Traversal() inserts := []TraversalInput{&tt.data} - fmt.Printf("inserts: %v\n", inserts) traversal := vertexTraversal(&g, inserts) // This is ugly but doesn't need to write to the DB // This just makes sure the traversal is correctly returned with the correct values diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 519f1bb1..36556101 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -1,8 +1,6 @@ package vertex import ( - "fmt" - "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -29,7 +27,6 @@ func (v Node) Traversal() VertexTraversal { g := source.GetGraphTraversal() for _, w := range inserts { data := w.(*graph.Node) - fmt.Printf("QWERTY: %v", data) g = g.AddV(v.Label()). Property("storeId", data.StoreId). Property("name", data.Name). diff --git a/pkg/kubehound/ingestor/pipeline/node_ingest.go b/pkg/kubehound/ingestor/pipeline/node_ingest.go index 2fc3fd75..34e7630a 100644 --- a/pkg/kubehound/ingestor/pipeline/node_ingest.go +++ b/pkg/kubehound/ingestor/pipeline/node_ingest.go @@ -7,7 +7,6 @@ import ( "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" "github.com/DataDog/KubeHound/pkg/kubehound/storage/cache" "github.com/DataDog/KubeHound/pkg/kubehound/store/collections" - "github.com/DataDog/KubeHound/pkg/telemetry/log" ) const ( @@ -46,7 +45,6 @@ func (i *NodeIngest) Initialize(ctx context.Context, deps *Dependencies) error { // streamCallback is invoked by the collector for each node collected. // The function ingests an input node into the cache/store/graph databases asynchronously. func (i *NodeIngest) IngestNode(ctx context.Context, node types.NodeType) error { - log.I.Infof("+++++++ INGEST NODE: (%s) ", node.Name) // Normalize node to store object format o, err := i.r.storeConvert.Node(ctx, node) if err != nil { @@ -84,7 +82,6 @@ func (i *NodeIngest) Complete(ctx context.Context) error { } func (i *NodeIngest) Run(ctx context.Context) error { - log.I.Infof("!!!!!! run NodeIngest !!!!!!!") return i.r.collect.StreamNodes(ctx, i) } diff --git a/pkg/kubehound/ingestor/pipeline_ingestor.go b/pkg/kubehound/ingestor/pipeline_ingestor.go index dc3d97e6..ec9cb84d 100644 --- a/pkg/kubehound/ingestor/pipeline_ingestor.go +++ b/pkg/kubehound/ingestor/pipeline_ingestor.go @@ -127,7 +127,6 @@ func (i PipelineIngestor) Run(outer context.Context) error { err := s.Run(ctx, deps) if err != nil { - log.I.Errorf("!!!!!!!!!! failed to run pipeline sequence thing !!!!!!!!: %v", err) l.Errorf("ingestor sequence %s run: %v", s.Name, err) // cancelAll(err) } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index be1b9004..8aefbad5 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -13,14 +13,15 @@ import ( var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) type JanusGraphAsyncEdgeWriter struct { - gremlin edge.EdgeTraversal - transaction *gremlingo.Transaction - traversalSource *gremlingo.GraphTraversalSource - inserts []any - consumerChan chan []any - writingInFligth sync.WaitGroup - batchSize int // Shouldn't this be "per edge types" ? - mu sync.Mutex + gremlin edge.EdgeTraversal + transaction *gremlingo.Transaction + traversalSource *gremlingo.GraphTraversalSource + inserts []any + consumerChan chan []any + writingInFligth sync.WaitGroup + batchSize int // Shouldn't this be "per edge types" ? + mu sync.Mutex + isTransactionEnabled bool } func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, e edge.Builder, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { @@ -30,17 +31,22 @@ func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemo opt(options) } - traversal := gremlingo.Traversal_().WithRemote(drc) - // tx := traversal.Tx() - // gtx, err := tx.Begin() - // if err != nil { - // return nil, err - // } + source := gremlingo.Traversal_().WithRemote(drc) + // quick switch to enable / disable transaction + if options.isTransactionEnabled { + log.I.Info("GraphDB transaction enabled!") + tx := source.Tx() + var err error + source, err = tx.Begin() + if err != nil { + return nil, err + } + } + jw := JanusGraphAsyncEdgeWriter{ - gremlin: e.Traversal(), - inserts: make([]any, 0), - // transaction: tx, - traversalSource: traversal, + gremlin: e.Traversal(), + inserts: make([]any, 0), + traversalSource: source, batchSize: 1, consumerChan: make(chan []any, 10000), } @@ -63,7 +69,7 @@ func (jge *JanusGraphAsyncEdgeWriter) backgroundWriter(ctx context.Context) { log.I.Errorf("failed to write data in background batch writer: %v", err) } case <-ctx.Done(): - log.I.Info("Closed background mongodb worker") + log.I.Info("Closed background janusgraph worker") return } } @@ -71,7 +77,7 @@ func (jge *JanusGraphAsyncEdgeWriter) backgroundWriter(ctx context.Context) { } func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any) error { - log.I.Infof("batch write JanusGraphAsyncEdgeWriter") + log.I.Debugf("batch write JanusGraphAsyncEdgeWriter with %d elements", len(data)) jge.writingInFligth.Add(1) defer jge.writingInFligth.Done() @@ -87,20 +93,26 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any promise := op.Iterate() err := <-promise if err != nil { - // jge.transaction.Rollback() + if jge.isTransactionEnabled { + jge.transaction.Rollback() + } return err } - // err = jge.transaction.Commit() - // if err != nil { - // log.I.Errorf("failed to commit: %+v", err) - // return err - // } - log.I.Infof("=== DONE == batch write JanusGraphAsyncEdgeWriter") + + if jge.isTransactionEnabled { + err = jge.transaction.Commit() + if err != nil { + log.I.Errorf("failed to commit: %+v", err) + return err + } + } return nil } func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { - // return e.transaction.Close() + if e.isTransactionEnabled { + return e.transaction.Close() + } return nil } @@ -120,7 +132,7 @@ func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { e.writingInFligth.Wait() return nil } - log.I.Infof("Flushing remaining of queue for edges: %+v", e.inserts) + log.I.Debugf("Flushing remaining of queue for edges: %+v", e.inserts) err := e.batchWrite(ctx, e.inserts) if err != nil { log.I.Errorf("Failed to batch write edge: %+v", err) @@ -144,6 +156,5 @@ func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { e.inserts = nil } e.inserts = append(e.inserts, edge) - log.I.Errorf("INSERTS AFTER APPEND (edge): %+v", &e.inserts) return nil } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index 728bc4b4..2afb01c5 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -16,14 +16,15 @@ var _ AsyncVertexWriter = (*JanusGraphAsyncVertexWriter)(nil) // type GremlinTraversalEdge func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal type JanusGraphAsyncVertexWriter struct { - gremlin vertex.VertexTraversal - transaction *gremlingo.Transaction - traversalSource *gremlingo.GraphTraversalSource - inserts []any - consumerChan chan []any - writingInFligth sync.WaitGroup - batchSize int // Shouldn't this be "per vertex types" ? - mu sync.Mutex + gremlin vertex.VertexTraversal + transaction *gremlingo.Transaction + traversalSource *gremlingo.GraphTraversalSource + inserts []any + consumerChan chan []any + writingInFligth sync.WaitGroup + batchSize int // Shouldn't this be "per vertex types" ? + mu sync.Mutex + isTransactionEnabled bool } func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, v vertex.Builder, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { @@ -34,15 +35,20 @@ func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRe } source := gremlingo.Traversal_().WithRemote(drc) - // tx := source.Tx() - // gtx, err := tx.Begin() - // if err != nil { - // return nil, err - // } + // quick switch to enable / disable transaction + if options.isTransactionEnabled { + log.I.Info("GraphDB transaction enabled!") + tx := source.Tx() + var err error + source, err = tx.Begin() + if err != nil { + return nil, err + } + } + jw := JanusGraphAsyncVertexWriter{ - gremlin: v.Traversal(), - inserts: make([]interface{}, 0), - // transaction: tx, + gremlin: v.Traversal(), + inserts: make([]interface{}, 0), traversalSource: source, batchSize: 1, consumerChan: make(chan []any, 10000), @@ -66,7 +72,7 @@ func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { log.I.Errorf("failed to write data in background batch writer: %v", err) } case <-ctx.Done(): - log.I.Info("Closed background mongodb worker") + log.I.Info("Closed background janusgraph worker") return } } @@ -74,7 +80,7 @@ func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { } func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []any) error { - log.I.Infof("batch write JanusGraphAsyncVertexWriter") + log.I.Debugf("batch write JanusGraphAsyncVertexWriter with %d elements", len(data)) jgv.writingInFligth.Add(1) defer jgv.writingInFligth.Done() @@ -82,31 +88,30 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a for _, d := range data { convertedToTraversalInput = append(convertedToTraversalInput, d.(vertex.TraversalInput)) } - - log.I.Infof("BEFORE gremlin(), traversal input: %+v", &convertedToTraversalInput) op := jgv.gremlin(jgv.traversalSource, convertedToTraversalInput) - log.I.Infof("BEFORE ITERATE") promise := op.Iterate() - log.I.Infof("BEFORE PROMISE") err := <-promise - log.I.Infof("AFTER PROMISE: %v, convertedToTraversalInput: %+v", err, convertedToTraversalInput) if err != nil { - log.I.Infof("IS THERE AN ERROR HERE: %+v", err) - // jgv.transaction.Rollback() + if jgv.isTransactionEnabled { + jgv.transaction.Rollback() + } return err } - // log.I.Infof("commiting work") - // err = jgv.transaction.Commit() - // if err != nil { - // log.I.Errorf("failed to commit: %+v", err) - // return err - // } - log.I.Infof("=== DONE == batch write JanusGraphAsyncVertexWriter") + if jgv.isTransactionEnabled { + log.I.Infof("commiting work") + err = jgv.transaction.Commit() + if err != nil { + log.I.Errorf("failed to commit: %+v", err) + return err + } + } return nil } func (v *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { - // return v.transaction.Close() + if v.isTransactionEnabled { + return v.transaction.Close() + } return nil } @@ -126,7 +131,7 @@ func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { v.writingInFligth.Wait() return nil } - log.I.Infof("Flushing remaining of queue for vertices: %+v", v.inserts) + err := v.batchWrite(ctx, v.inserts) if err != nil { log.I.Errorf("Failed to batch write vertex: %+v", err) diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index 0438e053..f0d33f24 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -10,6 +10,7 @@ import ( ) type writerOptions struct { + isTransactionEnabled bool } type WriterOption func(*writerOptions) diff --git a/test/system/setup_test.go b/test/system/setup_test.go index 6dc2424b..9dafece1 100644 --- a/test/system/setup_test.go +++ b/test/system/setup_test.go @@ -41,7 +41,7 @@ func runKubeHound() error { cmd := exec.CommandContext(cmdCtx, CollectorScriptPath, CollectorOutputDir) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - // defer cleanupCollected() + defer cleanupCollected() if err := cmd.Run(); err != nil { return fmt.Errorf("collector script execution: %v", err) } From 5e1181c54b39e32cbc00d947518737bfe4617190 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 12:53:17 +0200 Subject: [PATCH 39/61] Add tests --- pkg/kubehound/graph/vertex/container_test.go | 6 ++ pkg/kubehound/graph/vertex/identity_test.go | 53 +++++++++++++++++ pkg/kubehound/graph/vertex/node_test.go | 53 +++++++++++++++++ pkg/kubehound/graph/vertex/pod_test.go | 58 +++++++++++++++++++ pkg/kubehound/graph/vertex/role.go | 4 ++ pkg/kubehound/graph/vertex/role_test.go | 54 +++++++++++++++++ pkg/kubehound/graph/vertex/volume_test.go | 52 +++++++++++++++++ .../storage/graphdb/janusgraph_edge_writer.go | 1 - 8 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 pkg/kubehound/graph/vertex/identity_test.go create mode 100644 pkg/kubehound/graph/vertex/node_test.go create mode 100644 pkg/kubehound/graph/vertex/pod_test.go create mode 100644 pkg/kubehound/graph/vertex/role_test.go create mode 100644 pkg/kubehound/graph/vertex/volume_test.go diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go index 5c8dc162..1216184b 100644 --- a/pkg/kubehound/graph/vertex/container_test.go +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -59,10 +59,16 @@ func TestContainer_Traversal(t *testing.T) { // This just makes sure the traversal is correctly returned with the correct values assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "image") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "/usr/bin/sleep") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "600") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "lol2ndarguments") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "1234") assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "1337") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "NET_CAP_ADMIN") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "NET_RAW_ADMIN") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test pod") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test node") }) } } diff --git a/pkg/kubehound/graph/vertex/identity_test.go b/pkg/kubehound/graph/vertex/identity_test.go new file mode 100644 index 00000000..ab768ef0 --- /dev/null +++ b/pkg/kubehound/graph/vertex/identity_test.go @@ -0,0 +1,53 @@ +package vertex + +import ( + "fmt" + "testing" + + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + "github.com/stretchr/testify/assert" +) + +func TestIdentity_Traversal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want VertexTraversal + data graph.Identity + }{ + { + name: "Add Identities in JanusGraph", + // We set the values to all field with non default values + // so we are sure all are correctly propagated. + data: graph.Identity{ + StoreId: "test id", + Name: "test name identity", + IsNamespaced: true, + Namespace: "lol namespace", + Type: "some type", + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + v := Identity{} + + g := gremlingo.GraphTraversalSource{} + + vertexTraversal := v.Traversal() + inserts := []TraversalInput{&tt.data} + + traversal := vertexTraversal(&g, inserts) + // This is ugly but doesn't need to write to the DB + // This just makes sure the traversal is correctly returned with the correct values + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name identity") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "lol namespace") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "some type") + }) + } +} diff --git a/pkg/kubehound/graph/vertex/node_test.go b/pkg/kubehound/graph/vertex/node_test.go new file mode 100644 index 00000000..448d30ec --- /dev/null +++ b/pkg/kubehound/graph/vertex/node_test.go @@ -0,0 +1,53 @@ +package vertex + +import ( + "fmt" + "testing" + + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + "github.com/stretchr/testify/assert" +) + +func TestNode_Traversal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want VertexTraversal + data graph.Node + }{ + { + name: "Add Identities in JanusGraph", + // We set the values to all field with non default values + // so we are sure all are correctly propagated. + data: graph.Node{ + StoreId: "test id", + Name: "test name node", + IsNamespaced: true, + Namespace: "lol namespace", + Compromised: 1, + Critical: true, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + v := Node{} + + g := gremlingo.GraphTraversalSource{} + + vertexTraversal := v.Traversal() + inserts := []TraversalInput{&tt.data} + + traversal := vertexTraversal(&g, inserts) + // This is ugly but doesn't need to write to the DB + // This just makes sure the traversal is correctly returned with the correct values + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name node") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "lol namespace") + }) + } +} diff --git a/pkg/kubehound/graph/vertex/pod_test.go b/pkg/kubehound/graph/vertex/pod_test.go new file mode 100644 index 00000000..b21f4149 --- /dev/null +++ b/pkg/kubehound/graph/vertex/pod_test.go @@ -0,0 +1,58 @@ +package vertex + +import ( + "fmt" + "testing" + + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + "github.com/stretchr/testify/assert" +) + +func TestPod_Traversal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want VertexTraversal + data graph.Pod + }{ + { + name: "Add Identities in JanusGraph", + // We set the values to all field with non default values + // so we are sure all are correctly propagated. + data: graph.Pod{ + StoreId: "test id", + Name: "test name pod", + IsNamespaced: true, + Namespace: "lol namespace", + Compromised: 1, + Critical: true, + SharedProcessNamespace: true, + ServiceAccount: "some service account", + Node: "lol node", + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + v := Pod{} + + g := gremlingo.GraphTraversalSource{} + + vertexTraversal := v.Traversal() + inserts := []TraversalInput{&tt.data} + + traversal := vertexTraversal(&g, inserts) + // This is ugly but doesn't need to write to the DB + // This just makes sure the traversal is correctly returned with the correct values + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name pod") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "lol namespace") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "some service account") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "lol node") + }) + } +} diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index 67c9366f..d81d5024 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -3,6 +3,7 @@ package vertex import ( "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" gremlin "github.com/apache/tinkerpop/gremlin-go/driver" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) const ( @@ -32,6 +33,9 @@ func (v Role) Traversal() VertexTraversal { Property("name", data.Name). Property("is_namespaced", data.IsNamespaced). Property("namespace", data.Namespace) + for _, rule := range data.Rules { + g = g.Property(gremlingo.Cardinality.Set, "rules", rule) + } } return g } diff --git a/pkg/kubehound/graph/vertex/role_test.go b/pkg/kubehound/graph/vertex/role_test.go new file mode 100644 index 00000000..d682074c --- /dev/null +++ b/pkg/kubehound/graph/vertex/role_test.go @@ -0,0 +1,54 @@ +package vertex + +import ( + "fmt" + "testing" + + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + "github.com/stretchr/testify/assert" +) + +func TestRole_Traversal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want VertexTraversal + data graph.Role + }{ + { + name: "Add Identities in JanusGraph", + // We set the values to all field with non default values + // so we are sure all are correctly propagated. + data: graph.Role{ + StoreId: "test id", + Name: "test name role", + IsNamespaced: true, + Namespace: "lol namespace", + Rules: []string{"rule1", "rule2"}, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + v := Role{} + + g := gremlingo.GraphTraversalSource{} + + vertexTraversal := v.Traversal() + inserts := []TraversalInput{&tt.data} + + traversal := vertexTraversal(&g, inserts) + // This is ugly but doesn't need to write to the DB + // This just makes sure the traversal is correctly returned with the correct values + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name role") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "lol namespace") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "rule1") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "rule2") + }) + } +} diff --git a/pkg/kubehound/graph/vertex/volume_test.go b/pkg/kubehound/graph/vertex/volume_test.go new file mode 100644 index 00000000..0b9bcb6d --- /dev/null +++ b/pkg/kubehound/graph/vertex/volume_test.go @@ -0,0 +1,52 @@ +package vertex + +import ( + "fmt" + "testing" + + "github.com/DataDog/KubeHound/pkg/kubehound/models/graph" + gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" + "github.com/stretchr/testify/assert" +) + +func TestVolume_Traversal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want VertexTraversal + data graph.Volume + }{ + { + name: "Add Identities in JanusGraph", + // We set the values to all field with non default values + // so we are sure all are correctly propagated. + data: graph.Volume{ + StoreId: "test id", + Name: "test name volume", + Type: "test type", + Path: "some path", + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + v := Volume{} + + g := gremlingo.GraphTraversalSource{} + + vertexTraversal := v.Traversal() + inserts := []TraversalInput{&tt.data} + + traversal := vertexTraversal(&g, inserts) + // This is ugly but doesn't need to write to the DB + // This just makes sure the traversal is correctly returned with the correct values + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test id") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test name volume") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "test type") + assert.Contains(t, fmt.Sprintf("%s", traversal.Bytecode), "some path") + }) + } +} diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index 8aefbad5..9ae72a33 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -132,7 +132,6 @@ func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { e.writingInFligth.Wait() return nil } - log.I.Debugf("Flushing remaining of queue for edges: %+v", e.inserts) err := e.batchWrite(ctx, e.inserts) if err != nil { log.I.Errorf("Failed to batch write edge: %+v", err) From d53e7b511e8b4b9a8cdaa25300ba13ba7fc2de41 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 13:12:53 +0200 Subject: [PATCH 40/61] cleanup --- pkg/kubehound/graph/builder.go | 1 - pkg/kubehound/ingestor/pipeline/pod_ingest.go | 2 +- pkg/kubehound/ingestor/pipeline_ingestor.go | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/kubehound/graph/builder.go b/pkg/kubehound/graph/builder.go index 7e915828..f19ec15d 100644 --- a/pkg/kubehound/graph/builder.go +++ b/pkg/kubehound/graph/builder.go @@ -47,7 +47,6 @@ func (b *Builder) HealthCheck(ctx context.Context) error { // buildEdge inserts a class of edges into the graph database. // NOTE: function is blocking and expected to be called from within a goroutine. func (b *Builder) buildEdge(ctx context.Context, e edge.Builder) error { - log.I.Infof("entering buildEdge with edge.Builder: %+v", e) w, err := b.graphdb.EdgeWriter(ctx, e) if err != nil { return err diff --git a/pkg/kubehound/ingestor/pipeline/pod_ingest.go b/pkg/kubehound/ingestor/pipeline/pod_ingest.go index eb1dcc68..dd8aa6e8 100644 --- a/pkg/kubehound/ingestor/pipeline/pod_ingest.go +++ b/pkg/kubehound/ingestor/pipeline/pod_ingest.go @@ -109,7 +109,7 @@ func (i *PodIngest) processVolume(ctx context.Context, parent *store.Pod, volume // Normalize volume to store object format sv, err := i.r.storeConvert.Volume(ctx, volume, parent) if err != nil { - log.I.Errorf("CHANGEME: failed to process volume type: %v", err) + log.I.Errorf("failed to process volume type: %v (continuing)", err) return nil } diff --git a/pkg/kubehound/ingestor/pipeline_ingestor.go b/pkg/kubehound/ingestor/pipeline_ingestor.go index ec9cb84d..3e9a1d09 100644 --- a/pkg/kubehound/ingestor/pipeline_ingestor.go +++ b/pkg/kubehound/ingestor/pipeline_ingestor.go @@ -128,7 +128,7 @@ func (i PipelineIngestor) Run(outer context.Context) error { err := s.Run(ctx, deps) if err != nil { l.Errorf("ingestor sequence %s run: %v", s.Name, err) - // cancelAll(err) + cancelAll(err) } }() } From 11ffc8049b74afef205dc1d0f324554af8082d14 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 13:18:43 +0200 Subject: [PATCH 41/61] linter / cleanup --- pkg/kubehound/graph/vertex/container.go | 2 +- pkg/kubehound/graph/vertex/container_test.go | 2 +- pkg/kubehound/graph/vertex/identity.go | 2 +- pkg/kubehound/graph/vertex/identity_test.go | 2 +- pkg/kubehound/graph/vertex/node.go | 2 +- pkg/kubehound/graph/vertex/node_test.go | 2 +- pkg/kubehound/graph/vertex/pod.go | 2 +- pkg/kubehound/graph/vertex/pod_test.go | 2 +- pkg/kubehound/graph/vertex/role.go | 2 +- pkg/kubehound/graph/vertex/role_test.go | 2 +- pkg/kubehound/graph/vertex/volume.go | 2 +- pkg/kubehound/graph/vertex/volume_test.go | 2 +- pkg/kubehound/models/converter/graph.go | 12 ++--- pkg/kubehound/models/graph/models.go | 14 +++--- pkg/utils/interfaces.go | 50 -------------------- test/system/setup_test.go | 9 ---- 16 files changed, 25 insertions(+), 84 deletions(-) delete mode 100644 pkg/utils/interfaces.go diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 42c72adb..034256a2 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -29,7 +29,7 @@ func (v Container) Traversal() VertexTraversal { for _, insert := range inserts { i := insert.(*graph.Container) g = g.AddV(v.Label()). - Property("storeId", i.StoreId). + Property("storeId", i.StoreID). Property("name", i.Name). Property("image", i.Image). Property("privileged", i.Privileged). diff --git a/pkg/kubehound/graph/vertex/container_test.go b/pkg/kubehound/graph/vertex/container_test.go index 1216184b..22b2018c 100644 --- a/pkg/kubehound/graph/vertex/container_test.go +++ b/pkg/kubehound/graph/vertex/container_test.go @@ -22,7 +22,7 @@ func TestContainer_Traversal(t *testing.T) { // We set the values to all field with non default values // so we are sure all are correctly propagated. data: graph.Container{ - StoreId: "test id", + StoreID: "test id", Name: "test name", Image: "image", Command: []string{"/usr/bin/sleep"}, diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 932ab2da..62ae0b35 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -28,7 +28,7 @@ func (v Identity) Traversal() VertexTraversal { for _, i := range inserts { data := i.(*graph.Identity) g = g.AddV(v.Label()). - Property("storeId", data.StoreId). + Property("storeId", data.StoreID). Property("name", data.Name). Property("isNamespaced", data.IsNamespaced). Property("namespace", data.Namespace). diff --git a/pkg/kubehound/graph/vertex/identity_test.go b/pkg/kubehound/graph/vertex/identity_test.go index ab768ef0..18f3a5ce 100644 --- a/pkg/kubehound/graph/vertex/identity_test.go +++ b/pkg/kubehound/graph/vertex/identity_test.go @@ -22,7 +22,7 @@ func TestIdentity_Traversal(t *testing.T) { // We set the values to all field with non default values // so we are sure all are correctly propagated. data: graph.Identity{ - StoreId: "test id", + StoreID: "test id", Name: "test name identity", IsNamespaced: true, Namespace: "lol namespace", diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 36556101..17946dae 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -28,7 +28,7 @@ func (v Node) Traversal() VertexTraversal { for _, w := range inserts { data := w.(*graph.Node) g = g.AddV(v.Label()). - Property("storeId", data.StoreId). + Property("storeId", data.StoreID). Property("name", data.Name). Property("is_namespaced", data.IsNamespaced). Property("namespace", data.Namespace). diff --git a/pkg/kubehound/graph/vertex/node_test.go b/pkg/kubehound/graph/vertex/node_test.go index 448d30ec..9d8719c3 100644 --- a/pkg/kubehound/graph/vertex/node_test.go +++ b/pkg/kubehound/graph/vertex/node_test.go @@ -22,7 +22,7 @@ func TestNode_Traversal(t *testing.T) { // We set the values to all field with non default values // so we are sure all are correctly propagated. data: graph.Node{ - StoreId: "test id", + StoreID: "test id", Name: "test name node", IsNamespaced: true, Namespace: "lol namespace", diff --git a/pkg/kubehound/graph/vertex/pod.go b/pkg/kubehound/graph/vertex/pod.go index bdcbfad5..ac2aa9fb 100644 --- a/pkg/kubehound/graph/vertex/pod.go +++ b/pkg/kubehound/graph/vertex/pod.go @@ -28,7 +28,7 @@ func (v Pod) Traversal() VertexTraversal { for _, i := range inserts { data := i.(*graph.Pod) g = g.AddV(v.Label()). - Property("store_id", data.StoreId). + Property("store_id", data.StoreID). Property("name", data.Name). Property("is_namespaced", data.IsNamespaced). Property("namespace", data.Namespace). diff --git a/pkg/kubehound/graph/vertex/pod_test.go b/pkg/kubehound/graph/vertex/pod_test.go index b21f4149..7d0a863d 100644 --- a/pkg/kubehound/graph/vertex/pod_test.go +++ b/pkg/kubehound/graph/vertex/pod_test.go @@ -22,7 +22,7 @@ func TestPod_Traversal(t *testing.T) { // We set the values to all field with non default values // so we are sure all are correctly propagated. data: graph.Pod{ - StoreId: "test id", + StoreID: "test id", Name: "test name pod", IsNamespaced: true, Namespace: "lol namespace", diff --git a/pkg/kubehound/graph/vertex/role.go b/pkg/kubehound/graph/vertex/role.go index d81d5024..dea13c7b 100644 --- a/pkg/kubehound/graph/vertex/role.go +++ b/pkg/kubehound/graph/vertex/role.go @@ -29,7 +29,7 @@ func (v Role) Traversal() VertexTraversal { for _, i := range inserts { data := i.(*graph.Role) g = g.AddV(v.Label()). - Property("store_id", data.StoreId). + Property("store_id", data.StoreID). Property("name", data.Name). Property("is_namespaced", data.IsNamespaced). Property("namespace", data.Namespace) diff --git a/pkg/kubehound/graph/vertex/role_test.go b/pkg/kubehound/graph/vertex/role_test.go index d682074c..40762d23 100644 --- a/pkg/kubehound/graph/vertex/role_test.go +++ b/pkg/kubehound/graph/vertex/role_test.go @@ -22,7 +22,7 @@ func TestRole_Traversal(t *testing.T) { // We set the values to all field with non default values // so we are sure all are correctly propagated. data: graph.Role{ - StoreId: "test id", + StoreID: "test id", Name: "test name role", IsNamespaced: true, Namespace: "lol namespace", diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index 96f66920..010e6020 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -28,7 +28,7 @@ func (v Volume) Traversal() VertexTraversal { for _, i := range inserts { data := i.(*graph.Volume) g = g.AddV(v.Label()). - Property("storeId", data.StoreId). + Property("storeId", data.StoreID). Property("name", data.Name). Property("type", data.Type). Property("path", data.Path) diff --git a/pkg/kubehound/graph/vertex/volume_test.go b/pkg/kubehound/graph/vertex/volume_test.go index 0b9bcb6d..1df7c360 100644 --- a/pkg/kubehound/graph/vertex/volume_test.go +++ b/pkg/kubehound/graph/vertex/volume_test.go @@ -22,7 +22,7 @@ func TestVolume_Traversal(t *testing.T) { // We set the values to all field with non default values // so we are sure all are correctly propagated. data: graph.Volume{ - StoreId: "test id", + StoreID: "test id", Name: "test name volume", Type: "test type", Path: "some path", diff --git a/pkg/kubehound/models/converter/graph.go b/pkg/kubehound/models/converter/graph.go index 54285fb2..9c386313 100644 --- a/pkg/kubehound/models/converter/graph.go +++ b/pkg/kubehound/models/converter/graph.go @@ -21,7 +21,7 @@ func NewGraph() *GraphConverter { // Container returns the graph representation of a container vertex from a store container model input. func (c *GraphConverter) Container(input *store.Container) (*graph.Container, error) { output := &graph.Container{ - StoreId: input.Id.Hex(), + StoreID: input.Id.Hex(), Name: input.K8.Name, Image: input.K8.Image, Command: input.K8.Command, @@ -70,7 +70,7 @@ func (c *GraphConverter) Container(input *store.Container) (*graph.Container, er // Node returns the graph representation of a node vertex from a store node model input. func (c *GraphConverter) Node(input *store.Node) (*graph.Node, error) { output := &graph.Node{ - StoreId: input.Id.Hex(), + StoreID: input.Id.Hex(), Name: input.K8.Name, } @@ -85,7 +85,7 @@ func (c *GraphConverter) Node(input *store.Node) (*graph.Node, error) { // Pod returns the graph representation of a pod vertex from a store pod model input. func (c *GraphConverter) Pod(input *store.Pod) (*graph.Pod, error) { output := &graph.Pod{ - StoreId: input.Id.Hex(), + StoreID: input.Id.Hex(), Name: input.K8.Name, Namespace: input.K8.GetNamespace(), ServiceAccount: input.K8.Spec.ServiceAccountName, @@ -102,7 +102,7 @@ func (c *GraphConverter) Pod(input *store.Pod) (*graph.Pod, error) { // Volume returns the graph representation of a volume vertex from a store volume model input. func (c *GraphConverter) Volume(input *store.Volume) (*graph.Volume, error) { output := &graph.Volume{ - StoreId: input.Id.Hex(), + StoreID: input.Id.Hex(), Name: input.Name, } @@ -160,7 +160,7 @@ func (c *GraphConverter) flattenPolicyRules(input []rbacv1.PolicyRule) []string // Role returns the graph representation of a role vertex from a store role model input. func (c *GraphConverter) Role(input *store.Role) (*graph.Role, error) { output := &graph.Role{ - StoreId: input.Id.Hex(), + StoreID: input.Id.Hex(), Name: input.Name, Namespace: input.Namespace, Rules: c.flattenPolicyRules(input.Rules), @@ -172,7 +172,7 @@ func (c *GraphConverter) Role(input *store.Role) (*graph.Role, error) { // Identity returns the graph representation of an identity vertex from a store identity model input. func (c *GraphConverter) Identity(input *store.Identity) (*graph.Identity, error) { return &graph.Identity{ - StoreId: input.Id.Hex(), + StoreID: input.Id.Hex(), Name: input.Name, Namespace: input.Namespace, Type: input.Type, diff --git a/pkg/kubehound/models/graph/models.go b/pkg/kubehound/models/graph/models.go index 63d2bc2c..50dcff18 100644 --- a/pkg/kubehound/models/graph/models.go +++ b/pkg/kubehound/models/graph/models.go @@ -9,7 +9,7 @@ const ( ) type Container struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"store_id" mapstructure:"store_id"` Name string `json:"name" mapstructure:"name"` Image string `json:"image" mapstructure:"image"` Command []string `json:"command" mapstructure:"command"` @@ -35,7 +35,7 @@ type Group struct { } type Identity struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"store_id" mapstructure:"store_id"` Name string `json:"name" mapstructure:"name"` IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` @@ -43,7 +43,7 @@ type Identity struct { } type Node struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"store_id" mapstructure:"store_id"` Name string `json:"name" mapstructure:"name"` IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` @@ -52,7 +52,7 @@ type Node struct { } type Pod struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"store_id" mapstructure:"store_id"` Name string `json:"name" mapstructure:"name"` IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` @@ -64,7 +64,7 @@ type Pod struct { } type Role struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"store_id" mapstructure:"store_id"` Name string `json:"name" mapstructure:"name"` IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` @@ -72,7 +72,7 @@ type Role struct { } type Token struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"store_id" mapstructure:"store_id"` Name string `json:"name" mapstructure:"name"` Type string `json:"type" mapstructure:"type"` Identity string `json:"identity" mapstructure:"identity"` @@ -87,7 +87,7 @@ const ( ) type Volume struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"store_id" mapstructure:"store_id"` Name string `json:"name" mapstructure:"name"` Type string `json:"type" mapstructure:"type"` Path string `json:"path" mapstructure:"path"` diff --git a/pkg/utils/interfaces.go b/pkg/utils/interfaces.go deleted file mode 100644 index 1e8ef49a..00000000 --- a/pkg/utils/interfaces.go +++ /dev/null @@ -1,50 +0,0 @@ -package utils - -import ( - "fmt" - - "github.com/DataDog/KubeHound/pkg/telemetry/log" -) - -// func AnySliceToStringSlice(in []any) []string { -// s := make([]string, len(in)) -// for i, v := range in { -// s[i] = fmt.Sprint(v) -// } -// return s -// } -// func AnySliceToIntSlice(in []int64) []int64 { -// s := make([]int, len(in)) -// for i, v := range in { -// s[i] = v -// } -// return s -// } - -func ToSliceOfAny[Tin any](s []Tin) []string { - result := make([]string, len(s)) - for i, v := range s { - result[i] = fmt.Sprint(v) - } - return result -} - -func ConvertSliceAnyToTyped[T any, Tin any](data []Tin) []T { - converted := make([]T, len(data)) - for _, d := range converted { - converted = append(converted, d) - } - return converted -} - -func ConvertToSliceMapAny[T any](inserts []T) []map[string]any { - toStore := make([]map[string]any, len(inserts)) - for _, currentStruct := range inserts { - m, err := StructToMap(currentStruct) - if err != nil { - log.I.Errorf("Failed to convert struct to map for Nodes: %+v", err) - } - toStore = append(toStore, m) - } - return toStore -} diff --git a/test/system/setup_test.go b/test/system/setup_test.go index 9dafece1..a2041564 100644 --- a/test/system/setup_test.go +++ b/test/system/setup_test.go @@ -63,12 +63,3 @@ func TestMain(m *testing.M) { // Run the test suite os.Exit(m.Run()) } - -// func TestDebugE(m *testing.T) { -// if err := runKubeHound(); err != nil { -// log.I.Fatalf(err.Error()) -// } - -// // Run the test suite -// // os.Exit(m.Run()) -// } From 71acaa1e09818b192d7de8453d6a995c4759c814 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 13:41:53 +0200 Subject: [PATCH 42/61] Updates --- pkg/kubehound/graph/edge/container_attach.go | 8 ++++++- .../models/converter/converter_test.go | 16 ++++++------- pkg/kubehound/models/graph/models.go | 24 +++++++++---------- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/pkg/kubehound/graph/edge/container_attach.go b/pkg/kubehound/graph/edge/container_attach.go index 30677aa8..22c02237 100644 --- a/pkg/kubehound/graph/edge/container_attach.go +++ b/pkg/kubehound/graph/edge/container_attach.go @@ -34,10 +34,16 @@ func (e ContainerAttach) BatchSize() int { func (e ContainerAttach) Traversal() EdgeTraversal { return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - log.I.Errorf("CONVERT ME TO SOMETHING TYPED OTHERWISE THIS WILL BREAK") + log.I.Errorf("%s edge traversal does not work, inserting dummy input", e.Label()) g := source.GetGraphTraversal() + // return g.Inject(inserts).Unfold().As("ca"). + // V().HasLabel("Pod").Has("storeID", gremlin.T__.Select("ca").Select("pod")).As("pod"). + // V().HasLabel("Container").Has("storeID", gremlin.T__.Select("ca").Select("container")).As("container"). + // MergeE(e.Label()).From("pod").To("container") + g.AddV("dummy_entry") + // for _, i := range inserts { // data := i.(*ContainerAttachGroup) // podId, err := data.Pod.MarshalJSON() diff --git a/pkg/kubehound/models/converter/converter_test.go b/pkg/kubehound/models/converter/converter_test.go index 7ae2d6a9..0b4512b0 100644 --- a/pkg/kubehound/models/converter/converter_test.go +++ b/pkg/kubehound/models/converter/converter_test.go @@ -50,7 +50,7 @@ func TestConverter_NodePipeline(t *testing.T) { graphNode, err := NewGraph().Node(storeNode) assert.NoError(t, err, "graph node convert error") - assert.Equal(t, storeNode.Id.Hex(), graphNode.StoreId) + assert.Equal(t, storeNode.Id.Hex(), graphNode.StoreID) assert.Equal(t, storeNode.K8.Name, graphNode.Name) assert.False(t, storeNode.IsNamespaced) assert.Equal(t, storeNode.K8.Namespace, graphNode.Namespace) @@ -77,7 +77,7 @@ func TestConverter_RolePipeline(t *testing.T) { graphRole, err := NewGraph().Role(storeRole) assert.NoError(t, err, "graph role convert error") - assert.Equal(t, storeRole.Id.Hex(), graphRole.StoreId) + assert.Equal(t, storeRole.Id.Hex(), graphRole.StoreID) assert.Equal(t, storeRole.Name, graphRole.Name) assert.Equal(t, storeRole.Namespace, graphRole.Namespace) @@ -109,7 +109,7 @@ func TestConverter_ClusterRolePipeline(t *testing.T) { graphRole, err := NewGraph().Role(storeRole) assert.NoError(t, err, "graph role convert error") - assert.Equal(t, storeRole.Id.Hex(), graphRole.StoreId) + assert.Equal(t, storeRole.Id.Hex(), graphRole.StoreID) assert.Equal(t, storeRole.Name, graphRole.Name) assert.Equal(t, storeRole.Namespace, graphRole.Namespace) @@ -160,7 +160,7 @@ func TestConverter_RoleBindingPipeline(t *testing.T) { graphIdentity, err := NewGraph().Identity(storeIdentity) assert.NoError(t, err, "graph role binding convert error") - assert.Equal(t, storeIdentity.Id.Hex(), graphIdentity.StoreId) + assert.Equal(t, storeIdentity.Id.Hex(), graphIdentity.StoreID) assert.Equal(t, storeIdentity.Name, graphIdentity.Name) assert.Equal(t, storeIdentity.Namespace, graphIdentity.Namespace) assert.Equal(t, storeIdentity.Type, graphIdentity.Type) @@ -204,7 +204,7 @@ func TestConverter_ClusterRoleBindingPipeline(t *testing.T) { graphIdentity, err := NewGraph().Identity(storeIdentity) assert.NoError(t, err, "graph role binding convert error") - assert.Equal(t, storeIdentity.Id.Hex(), graphIdentity.StoreId) + assert.Equal(t, storeIdentity.Id.Hex(), graphIdentity.StoreID) assert.Equal(t, storeIdentity.Name, graphIdentity.Name) assert.Equal(t, storeIdentity.Namespace, graphIdentity.Namespace) assert.Equal(t, storeIdentity.Type, graphIdentity.Type) @@ -253,7 +253,7 @@ func TestConverter_PodPipeline(t *testing.T) { graphPod, err := NewGraph().Pod(storePod) assert.NoError(t, err, "graph pod convert error") - assert.Equal(t, storePod.Id.Hex(), graphPod.StoreId) + assert.Equal(t, storePod.Id.Hex(), graphPod.StoreID) assert.Equal(t, storePod.K8.Name, graphPod.Name) assert.Equal(t, storePod.K8.Namespace, graphPod.Namespace) assert.False(t, graphPod.SharedProcessNamespace) @@ -298,7 +298,7 @@ func TestConverter_PodChildPipeline(t *testing.T) { graphContainer, err := NewGraph().Container(storeContainer) assert.NoError(t, err, "graph container convert error") - assert.Equal(t, storeContainer.Id.Hex(), graphContainer.StoreId) + assert.Equal(t, storeContainer.Id.Hex(), graphContainer.StoreID) assert.Equal(t, storeContainer.K8.Name, graphContainer.Name) assert.Equal(t, storeContainer.K8.Image, graphContainer.Image) assert.Equal(t, storeContainer.K8.Command, graphContainer.Command) @@ -324,7 +324,7 @@ func TestConverter_PodChildPipeline(t *testing.T) { graphVolume, err := NewGraph().Volume(storeVolume) assert.NoError(t, err, "graph volume convert error") - assert.Equal(t, storeVolume.Id.Hex(), graphVolume.StoreId) + assert.Equal(t, storeVolume.Id.Hex(), graphVolume.StoreID) assert.Equal(t, storeVolume.Name, graphVolume.Name) assert.Equal(t, graph.VolumeTypeProjected, graphVolume.Type) assert.Equal(t, "token", graphVolume.Path) diff --git a/pkg/kubehound/models/graph/models.go b/pkg/kubehound/models/graph/models.go index 50dcff18..190de380 100644 --- a/pkg/kubehound/models/graph/models.go +++ b/pkg/kubehound/models/graph/models.go @@ -9,7 +9,7 @@ const ( ) type Container struct { - StoreID string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` Image string `json:"image" mapstructure:"image"` Command []string `json:"command" mapstructure:"command"` @@ -30,31 +30,31 @@ type Container struct { } type Group struct { - StoreId string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` } type Identity struct { - StoreID string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` - IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + IsNamespaced bool `json:"isNamespaced" mapstructure:"isNamespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` Type string `json:"type" mapstructure:"type"` } type Node struct { - StoreID string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` - IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + IsNamespaced bool `json:"isNamespaced" mapstructure:"isNamespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` Compromised CompromiseType `json:"compromised,omitempty" mapstructure:"compromised"` Critical bool `json:"critical,omitempty" mapstructure:"critical"` } type Pod struct { - StoreID string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` - IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + IsNamespaced bool `json:"isNamespaced" mapstructure:"isNamespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` SharedProcessNamespace bool `json:"sharedProcessNamespace" mapstructure:"sharedProcessNamespace"` ServiceAccount string `json:"serviceAccount" mapstructure:"serviceAccount"` @@ -64,15 +64,15 @@ type Pod struct { } type Role struct { - StoreID string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` - IsNamespaced bool `json:"is_namespaced" mapstructure:"is_namespaced"` + IsNamespaced bool `json:"isNamespaced" mapstructure:"isNamespaced"` Namespace string `json:"namespace" mapstructure:"namespace"` Rules []string `json:"rules" mapstructure:"rules"` } type Token struct { - StoreID string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` Type string `json:"type" mapstructure:"type"` Identity string `json:"identity" mapstructure:"identity"` @@ -87,7 +87,7 @@ const ( ) type Volume struct { - StoreID string `json:"store_id" mapstructure:"store_id"` + StoreID string `json:"storeID" mapstructure:"storeID"` Name string `json:"name" mapstructure:"name"` Type string `json:"type" mapstructure:"type"` Path string `json:"path" mapstructure:"path"` From cd53b732d55a021d2064d0d003edbdbbcdfb7470 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 14:05:54 +0200 Subject: [PATCH 43/61] cleanup options --- pkg/kubehound/ingestor/pipeline/ingest_resources.go | 6 +++++- .../storage/graphdb/janusgraph_edge_writer.go | 10 ++++++---- pkg/kubehound/storage/graphdb/janusgraph_provider.go | 5 ++++- .../storage/graphdb/janusgraph_vertex_writer.go | 10 ++++++---- pkg/kubehound/storage/graphdb/provider.go | 7 +++++++ 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/pkg/kubehound/ingestor/pipeline/ingest_resources.go b/pkg/kubehound/ingestor/pipeline/ingest_resources.go index d3b0714f..4ec31ffb 100644 --- a/pkg/kubehound/ingestor/pipeline/ingest_resources.go +++ b/pkg/kubehound/ingestor/pipeline/ingest_resources.go @@ -86,7 +86,11 @@ func WithStoreWriter[T collections.Collection](c T) IngestResourceOption { // To access the writer use the graphWriter(v vertex.Vertex) function. func WithGraphWriter(v vertex.Builder) IngestResourceOption { return func(ctx context.Context, rOpts *resourceOptions, deps *Dependencies) error { - w, err := deps.GraphDB.VertexWriter(ctx, v) + opts := []graphdb.WriterOption{ + // graphdb.WithTransaction(), // disable options for now + } + + w, err := deps.GraphDB.VertexWriter(ctx, v, opts...) if err != nil { return err } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index 9ae72a33..f51ea65d 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -19,7 +19,7 @@ type JanusGraphAsyncEdgeWriter struct { inserts []any consumerChan chan []any writingInFligth sync.WaitGroup - batchSize int // Shouldn't this be "per edge types" ? + batchSize int mu sync.Mutex isTransactionEnabled bool } @@ -33,9 +33,10 @@ func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemo source := gremlingo.Traversal_().WithRemote(drc) // quick switch to enable / disable transaction + var tx *gremlingo.Transaction if options.isTransactionEnabled { log.I.Info("GraphDB transaction enabled!") - tx := source.Tx() + tx = source.Tx() var err error source, err = tx.Begin() if err != nil { @@ -47,8 +48,9 @@ func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemo gremlin: e.Traversal(), inserts: make([]any, 0), traversalSource: source, - batchSize: 1, - consumerChan: make(chan []any, 10000), + transaction: tx, + batchSize: e.BatchSize(), + consumerChan: make(chan []any, e.BatchSize()*channelSizeBatchFactor), } jw.backgroundWriter(ctx) return &jw, nil diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 11450da5..11a3f07c 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -9,6 +9,9 @@ import ( gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) +// TODO maybe move that into a config file? +const channelSizeBatchFactor = 4 + var _ Provider = (*JanusGraphProvider)(nil) type JanusGraphProvider struct { @@ -58,7 +61,7 @@ func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Builde // EdgeWriter creates a new AsyncEdgeWriter instance to enable asynchronous bulk inserts of edges. func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Builder, opts ...WriterOption) (AsyncEdgeWriter, error) { - writer, err := NewJanusGraphAsyncEdgeWriter(ctx, jgp.client, e) + writer, err := NewJanusGraphAsyncEdgeWriter(ctx, jgp.client, e, opts...) if err != nil { return nil, err } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index 2afb01c5..5f0291e3 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -22,7 +22,7 @@ type JanusGraphAsyncVertexWriter struct { inserts []any consumerChan chan []any writingInFligth sync.WaitGroup - batchSize int // Shouldn't this be "per vertex types" ? + batchSize int mu sync.Mutex isTransactionEnabled bool } @@ -36,9 +36,10 @@ func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRe source := gremlingo.Traversal_().WithRemote(drc) // quick switch to enable / disable transaction + var tx *gremlingo.Transaction if options.isTransactionEnabled { log.I.Info("GraphDB transaction enabled!") - tx := source.Tx() + tx = source.Tx() var err error source, err = tx.Begin() if err != nil { @@ -49,9 +50,10 @@ func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRe jw := JanusGraphAsyncVertexWriter{ gremlin: v.Traversal(), inserts: make([]interface{}, 0), + transaction: tx, traversalSource: source, - batchSize: 1, - consumerChan: make(chan []any, 10000), + batchSize: v.BatchSize(), + consumerChan: make(chan []any, v.BatchSize()*channelSizeBatchFactor), } jw.backgroundWriter(ctx) return &jw, nil diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index f0d33f24..9f36dd41 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -15,6 +15,13 @@ type writerOptions struct { type WriterOption func(*writerOptions) +// WithConfigPath sets the path for the KubeHound config file. +func WithTransaction() WriterOption { + return func(opt *writerOptions) { + opt.isTransactionEnabled = true + } +} + // Provider defines the interface for implementations of the graphdb provider for storage of the calculated K8s attack graph. // //go:generate mockery --name Provider --output mocks --case underscore --filename graph_provider.go --with-expecter From de1d3801c518178ea15e9ce22523ec7343980c4d Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 14:07:29 +0200 Subject: [PATCH 44/61] casing again --- pkg/kubehound/graph/vertex/container.go | 2 +- pkg/kubehound/graph/vertex/identity.go | 2 +- pkg/kubehound/graph/vertex/node.go | 2 +- pkg/kubehound/graph/vertex/volume.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kubehound/graph/vertex/container.go b/pkg/kubehound/graph/vertex/container.go index 034256a2..f8c9be55 100644 --- a/pkg/kubehound/graph/vertex/container.go +++ b/pkg/kubehound/graph/vertex/container.go @@ -29,7 +29,7 @@ func (v Container) Traversal() VertexTraversal { for _, insert := range inserts { i := insert.(*graph.Container) g = g.AddV(v.Label()). - Property("storeId", i.StoreID). + Property("storeID", i.StoreID). Property("name", i.Name). Property("image", i.Image). Property("privileged", i.Privileged). diff --git a/pkg/kubehound/graph/vertex/identity.go b/pkg/kubehound/graph/vertex/identity.go index 62ae0b35..53dbc5cc 100644 --- a/pkg/kubehound/graph/vertex/identity.go +++ b/pkg/kubehound/graph/vertex/identity.go @@ -28,7 +28,7 @@ func (v Identity) Traversal() VertexTraversal { for _, i := range inserts { data := i.(*graph.Identity) g = g.AddV(v.Label()). - Property("storeId", data.StoreID). + Property("storeID", data.StoreID). Property("name", data.Name). Property("isNamespaced", data.IsNamespaced). Property("namespace", data.Namespace). diff --git a/pkg/kubehound/graph/vertex/node.go b/pkg/kubehound/graph/vertex/node.go index 17946dae..d173c6d4 100644 --- a/pkg/kubehound/graph/vertex/node.go +++ b/pkg/kubehound/graph/vertex/node.go @@ -28,7 +28,7 @@ func (v Node) Traversal() VertexTraversal { for _, w := range inserts { data := w.(*graph.Node) g = g.AddV(v.Label()). - Property("storeId", data.StoreID). + Property("storeID", data.StoreID). Property("name", data.Name). Property("is_namespaced", data.IsNamespaced). Property("namespace", data.Namespace). diff --git a/pkg/kubehound/graph/vertex/volume.go b/pkg/kubehound/graph/vertex/volume.go index 010e6020..25331a87 100644 --- a/pkg/kubehound/graph/vertex/volume.go +++ b/pkg/kubehound/graph/vertex/volume.go @@ -28,7 +28,7 @@ func (v Volume) Traversal() VertexTraversal { for _, i := range inserts { data := i.(*graph.Volume) g = g.AddV(v.Label()). - Property("storeId", data.StoreID). + Property("storeID", data.StoreID). Property("name", data.Name). Property("type", data.Type). Property("path", data.Path) From d4bf9a20d741af320d9aac1d3ab17b3ec5232be1 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 15:59:44 +0200 Subject: [PATCH 45/61] WIP PR comments --- Makefile | 2 +- .../storage/graphdb/janusgraph_edge_writer.go | 55 ++++++------- .../graphdb/janusgraph_vertex_writer.go | 78 ++++++++++--------- pkg/kubehound/storage/graphdb/provider.go | 6 +- pkg/kubehound/storage/storedb/mongo_writer.go | 10 +-- 5 files changed, 76 insertions(+), 75 deletions(-) diff --git a/Makefile b/Makefile index 78935853..a11527ee 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test: .PHONY: system-test system-test: - cd test/system && go test -v -timeout "30s" -race ./... + cd test/system && go test -v -timeout "60s" -race ./... .PHONY: local-cluster-create local-cluster-setup: diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index f51ea65d..63a029c7 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -46,18 +46,18 @@ func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemo jw := JanusGraphAsyncEdgeWriter{ gremlin: e.Traversal(), - inserts: make([]any, 0), + inserts: make([]any, 0, e.BatchSize()), traversalSource: source, transaction: tx, batchSize: e.BatchSize(), consumerChan: make(chan []any, e.BatchSize()*channelSizeBatchFactor), } - jw.backgroundWriter(ctx) + jw.startBackgroundWriter(ctx) return &jw, nil } -// backgroundWriter starts a background go routine -func (jge *JanusGraphAsyncEdgeWriter) backgroundWriter(ctx context.Context) { +// startBackgroundWriter starts a background go routine +func (jge *JanusGraphAsyncEdgeWriter) startBackgroundWriter(ctx context.Context) { go func() { for { select { @@ -66,6 +66,7 @@ func (jge *JanusGraphAsyncEdgeWriter) backgroundWriter(ctx context.Context) { if data == nil { return } + jge.writingInFligth.Add(1) err := jge.batchWrite(ctx, data) if err != nil { log.I.Errorf("failed to write data in background batch writer: %v", err) @@ -80,7 +81,6 @@ func (jge *JanusGraphAsyncEdgeWriter) backgroundWriter(ctx context.Context) { func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any) error { log.I.Debugf("batch write JanusGraphAsyncEdgeWriter with %d elements", len(data)) - jge.writingInFligth.Add(1) defer jge.writingInFligth.Done() // This seems ~pointless BUT is required to have the ability to use edge.TraversalInput/vertex.TraversalInput @@ -111,51 +111,54 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any return nil } -func (e *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { - if e.isTransactionEnabled { - return e.transaction.Close() +func (jge *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { + if jge.isTransactionEnabled { + return jge.transaction.Close() } return nil } -func (e *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { - e.mu.Lock() - defer e.mu.Unlock() +func (jge *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { + jge.mu.Lock() + defer jge.mu.Unlock() - if e.traversalSource == nil { + if jge.traversalSource == nil { return errors.New("JanusGraph traversalSource is not initialized") } - if len(e.inserts) == 0 { + if len(jge.inserts) == 0 { log.I.Debugf("Skipping flush on edges writer as no write operations left") // we need to send something to the channel from this function whenever we don't return an error // we cannot defer it because the go routine may last longer than the current function // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data - e.writingInFligth.Wait() + jge.writingInFligth.Wait() return nil } - err := e.batchWrite(ctx, e.inserts) + + jge.writingInFligth.Add(1) + err := jge.batchWrite(ctx, jge.inserts) if err != nil { log.I.Errorf("Failed to batch write edge: %+v", err) - e.writingInFligth.Wait() + jge.writingInFligth.Wait() return err } log.I.Info("Done flushing edges, clearing the queue") - e.inserts = nil + jge.inserts = nil return nil } -func (e *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { - e.mu.Lock() - defer e.mu.Unlock() - if len(e.inserts) > e.batchSize { - copied := make([]any, len(e.inserts)) - copy(copied, e.inserts) - e.consumerChan <- copied +func (jge *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { + jge.mu.Lock() + defer jge.mu.Unlock() + + jge.inserts = append(jge.inserts, edge) + if len(jge.inserts) > jge.batchSize { + copied := make([]any, len(jge.inserts)) + copy(copied, jge.inserts) + jge.consumerChan <- copied // cleanup the ops array after we have copied it to the channel - e.inserts = nil + jge.inserts = nil } - e.inserts = append(e.inserts, edge) return nil } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index 5f0291e3..b5a86f58 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -12,15 +12,12 @@ import ( var _ AsyncVertexWriter = (*JanusGraphAsyncVertexWriter)(nil) -// type GremlinTraversalVertex func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal -// type GremlinTraversalEdge func(*gremlingo.GraphTraversalSource, []any) *gremlingo.GraphTraversal - type JanusGraphAsyncVertexWriter struct { gremlin vertex.VertexTraversal transaction *gremlingo.Transaction traversalSource *gremlingo.GraphTraversalSource - inserts []any - consumerChan chan []any + inserts []vertex.TraversalInput + consumerChan chan []vertex.TraversalInput writingInFligth sync.WaitGroup batchSize int mu sync.Mutex @@ -49,18 +46,18 @@ func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRe jw := JanusGraphAsyncVertexWriter{ gremlin: v.Traversal(), - inserts: make([]interface{}, 0), + inserts: make([]vertex.TraversalInput, 0, v.BatchSize()), transaction: tx, traversalSource: source, batchSize: v.BatchSize(), - consumerChan: make(chan []any, v.BatchSize()*channelSizeBatchFactor), + consumerChan: make(chan []vertex.TraversalInput, v.BatchSize()*channelSizeBatchFactor), } - jw.backgroundWriter(ctx) + jw.startBackgroundWriter(ctx) return &jw, nil } -// backgroundWriter starts a background go routine -func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { +// startBackgroundWriter starts a background go routine +func (jgv *JanusGraphAsyncVertexWriter) startBackgroundWriter(ctx context.Context) { go func() { for { select { @@ -69,6 +66,7 @@ func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { if data == nil { return } + jgv.writingInFligth.Add(1) err := jgv.batchWrite(ctx, data) if err != nil { log.I.Errorf("failed to write data in background batch writer: %v", err) @@ -81,16 +79,15 @@ func (jgv *JanusGraphAsyncVertexWriter) backgroundWriter(ctx context.Context) { }() } -func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []any) error { +func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []vertex.TraversalInput) error { log.I.Debugf("batch write JanusGraphAsyncVertexWriter with %d elements", len(data)) - jgv.writingInFligth.Add(1) defer jgv.writingInFligth.Done() - convertedToTraversalInput := make([]vertex.TraversalInput, 0) - for _, d := range data { - convertedToTraversalInput = append(convertedToTraversalInput, d.(vertex.TraversalInput)) - } - op := jgv.gremlin(jgv.traversalSource, convertedToTraversalInput) + // convertedToTraversalInput := make([]vertex.TraversalInput, 0) + // for _, d := range data { + // convertedToTraversalInput = append(convertedToTraversalInput, d.(vertex.TraversalInput)) + // } + op := jgv.gremlin(jgv.traversalSource, data) promise := op.Iterate() err := <-promise if err != nil { @@ -110,52 +107,57 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []a return nil } -func (v *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { - if v.isTransactionEnabled { - return v.transaction.Close() +func (jgv *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { + if jgv.isTransactionEnabled { + return jgv.transaction.Close() } return nil } -func (v *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { - v.mu.Lock() - defer v.mu.Unlock() +// Flush triggers writes of any remaining items in the queue. +// This is blocking +func (jgv *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { + jgv.mu.Lock() + defer jgv.mu.Unlock() - if v.traversalSource == nil { + if jgv.traversalSource == nil { return errors.New("JanusGraph traversalSource is not initialized") } - if len(v.inserts) == 0 { + if len(jgv.inserts) == 0 { log.I.Debugf("Skipping flush on vertex writer as no write operations left") // we need to send something to the channel from this function whenever we don't return an error // we cannot defer it because the go routine may last longer than the current function // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data - v.writingInFligth.Wait() + jgv.writingInFligth.Wait() return nil } - err := v.batchWrite(ctx, v.inserts) + jgv.writingInFligth.Add(1) + err := jgv.batchWrite(ctx, jgv.inserts) if err != nil { log.I.Errorf("Failed to batch write vertex: %+v", err) - v.writingInFligth.Wait() + jgv.writingInFligth.Wait() return err } log.I.Info("Done flushing vertices, clearing the queue") - v.inserts = nil + jgv.inserts = nil return nil } -func (vw *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, vertex any) error { - vw.mu.Lock() - defer vw.mu.Unlock() - if len(vw.inserts) > vw.batchSize { - copied := make([]any, len(vw.inserts)) - copy(copied, vw.inserts) - vw.consumerChan <- copied +func (jgv *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, v any) error { + jgv.mu.Lock() + defer jgv.mu.Unlock() + + jgv.inserts = append(jgv.inserts, v) + if len(jgv.inserts) > jgv.batchSize { + var copied []vertex.TraversalInput + copied = make([]vertex.TraversalInput, len(jgv.inserts)) + copy(copied, jgv.inserts) + jgv.consumerChan <- copied // cleanup the ops array after we have copied it to the channel - vw.inserts = nil + jgv.inserts = nil } - vw.inserts = append(vw.inserts, vertex) return nil } diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index 9f36dd41..de70a688 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -71,9 +71,5 @@ type AsyncEdgeWriter interface { // Factory returns an initialized instance of a graphdb provider from the provided application config. func Factory(ctx context.Context, cfg *config.KubehoundConfig) (Provider, error) { - provider, err := NewGraphDriver(ctx, cfg.JanusGraph.URL) - if err != nil { - return nil, err - } - return provider, nil + return NewGraphDriver(ctx, cfg.JanusGraph.URL) } diff --git a/pkg/kubehound/storage/storedb/mongo_writer.go b/pkg/kubehound/storage/storedb/mongo_writer.go index 0e06ddfc..a235765f 100644 --- a/pkg/kubehound/storage/storedb/mongo_writer.go +++ b/pkg/kubehound/storage/storedb/mongo_writer.go @@ -13,7 +13,7 @@ import ( const ( // TODO: this might need to be adjusted in the future, potentially per type of collections - // We don't have the data yet, so lets just hardcode a small-ish value for now + // We don't have the data yet, so lets just hardcode a relatively high value for now consumerChanSize = 10000 ) @@ -35,12 +35,12 @@ func NewMongoAsyncWriter(ctx context.Context, mp *MongoProvider, collection coll batchSize: collection.BatchSize(), } maw.consumerChan = make(chan []mongo.WriteModel, consumerChanSize) - maw.backgroundWriter(ctx) + maw.startBackgroundWriter(ctx) return &maw } -// backgroundWriter starts a background go routine -func (maw *MongoAsyncWriter) backgroundWriter(ctx context.Context) { +// startBackgroundWriter starts a background go routine +func (maw *MongoAsyncWriter) startBackgroundWriter(ctx context.Context) { go func() { for { select { @@ -75,12 +75,12 @@ func (maw *MongoAsyncWriter) batchWrite(ctx context.Context, ops []mongo.WriteMo // Queue add a model to an asynchronous write queue. Non-blocking. func (maw *MongoAsyncWriter) Queue(ctx context.Context, model any) error { + maw.ops = append(maw.ops, mongo.NewInsertOneModel().SetDocument(model)) if len(maw.ops) > maw.batchSize { maw.consumerChan <- maw.ops // cleanup the ops array after we have copied it to the channel maw.ops = nil } - maw.ops = append(maw.ops, mongo.NewInsertOneModel().SetDocument(model)) return nil } From 1005e5539b3c69f3d2ccd14c3fdea89f1ae0c773 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 16:01:44 +0200 Subject: [PATCH 46/61] remove transactions janusgraph --- .../storage/graphdb/janusgraph_edge_writer.go | 41 +++--------------- .../graphdb/janusgraph_vertex_writer.go | 43 +++---------------- 2 files changed, 14 insertions(+), 70 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index 63a029c7..4ee4b2fa 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -13,15 +13,13 @@ import ( var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) type JanusGraphAsyncEdgeWriter struct { - gremlin edge.EdgeTraversal - transaction *gremlingo.Transaction - traversalSource *gremlingo.GraphTraversalSource - inserts []any - consumerChan chan []any - writingInFligth sync.WaitGroup - batchSize int - mu sync.Mutex - isTransactionEnabled bool + gremlin edge.EdgeTraversal + traversalSource *gremlingo.GraphTraversalSource + inserts []any + consumerChan chan []any + writingInFligth sync.WaitGroup + batchSize int + mu sync.Mutex } func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, e edge.Builder, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { @@ -32,23 +30,11 @@ func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemo } source := gremlingo.Traversal_().WithRemote(drc) - // quick switch to enable / disable transaction - var tx *gremlingo.Transaction - if options.isTransactionEnabled { - log.I.Info("GraphDB transaction enabled!") - tx = source.Tx() - var err error - source, err = tx.Begin() - if err != nil { - return nil, err - } - } jw := JanusGraphAsyncEdgeWriter{ gremlin: e.Traversal(), inserts: make([]any, 0, e.BatchSize()), traversalSource: source, - transaction: tx, batchSize: e.BatchSize(), consumerChan: make(chan []any, e.BatchSize()*channelSizeBatchFactor), } @@ -95,26 +81,13 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any promise := op.Iterate() err := <-promise if err != nil { - if jge.isTransactionEnabled { - jge.transaction.Rollback() - } return err } - if jge.isTransactionEnabled { - err = jge.transaction.Commit() - if err != nil { - log.I.Errorf("failed to commit: %+v", err) - return err - } - } return nil } func (jge *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { - if jge.isTransactionEnabled { - return jge.transaction.Close() - } return nil } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index b5a86f58..a4308e8b 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -13,15 +13,13 @@ import ( var _ AsyncVertexWriter = (*JanusGraphAsyncVertexWriter)(nil) type JanusGraphAsyncVertexWriter struct { - gremlin vertex.VertexTraversal - transaction *gremlingo.Transaction - traversalSource *gremlingo.GraphTraversalSource - inserts []vertex.TraversalInput - consumerChan chan []vertex.TraversalInput - writingInFligth sync.WaitGroup - batchSize int - mu sync.Mutex - isTransactionEnabled bool + gremlin vertex.VertexTraversal + traversalSource *gremlingo.GraphTraversalSource + inserts []vertex.TraversalInput + consumerChan chan []vertex.TraversalInput + writingInFligth sync.WaitGroup + batchSize int + mu sync.Mutex } func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, v vertex.Builder, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { @@ -32,22 +30,9 @@ func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRe } source := gremlingo.Traversal_().WithRemote(drc) - // quick switch to enable / disable transaction - var tx *gremlingo.Transaction - if options.isTransactionEnabled { - log.I.Info("GraphDB transaction enabled!") - tx = source.Tx() - var err error - source, err = tx.Begin() - if err != nil { - return nil, err - } - } - jw := JanusGraphAsyncVertexWriter{ gremlin: v.Traversal(), inserts: make([]vertex.TraversalInput, 0, v.BatchSize()), - transaction: tx, traversalSource: source, batchSize: v.BatchSize(), consumerChan: make(chan []vertex.TraversalInput, v.BatchSize()*channelSizeBatchFactor), @@ -91,26 +76,12 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []v promise := op.Iterate() err := <-promise if err != nil { - if jgv.isTransactionEnabled { - jgv.transaction.Rollback() - } return err } - if jgv.isTransactionEnabled { - log.I.Infof("commiting work") - err = jgv.transaction.Commit() - if err != nil { - log.I.Errorf("failed to commit: %+v", err) - return err - } - } return nil } func (jgv *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { - if jgv.isTransactionEnabled { - return jgv.transaction.Close() - } return nil } From 817d172febd4f754316599888b732d448b253e44 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 16:42:49 +0200 Subject: [PATCH 47/61] fix bugs --- .../storage/graphdb/janusgraph_edge_writer.go | 55 +++++++------------ .../graphdb/janusgraph_vertex_writer.go | 36 +++++------- 2 files changed, 34 insertions(+), 57 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index 4ee4b2fa..0f3a5b55 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -15,8 +15,8 @@ var _ AsyncEdgeWriter = (*JanusGraphAsyncEdgeWriter)(nil) type JanusGraphAsyncEdgeWriter struct { gremlin edge.EdgeTraversal traversalSource *gremlingo.GraphTraversalSource - inserts []any - consumerChan chan []any + inserts []edge.TraversalInput + consumerChan chan []edge.TraversalInput writingInFligth sync.WaitGroup batchSize int mu sync.Mutex @@ -33,10 +33,10 @@ func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemo jw := JanusGraphAsyncEdgeWriter{ gremlin: e.Traversal(), - inserts: make([]any, 0, e.BatchSize()), + inserts: make([]edge.TraversalInput, 0, e.BatchSize()), traversalSource: source, batchSize: e.BatchSize(), - consumerChan: make(chan []any, e.BatchSize()*channelSizeBatchFactor), + consumerChan: make(chan []edge.TraversalInput, e.BatchSize()*channelSizeBatchFactor), } jw.startBackgroundWriter(ctx) return &jw, nil @@ -52,7 +52,6 @@ func (jge *JanusGraphAsyncEdgeWriter) startBackgroundWriter(ctx context.Context) if data == nil { return } - jge.writingInFligth.Add(1) err := jge.batchWrite(ctx, data) if err != nil { log.I.Errorf("failed to write data in background batch writer: %v", err) @@ -65,19 +64,11 @@ func (jge *JanusGraphAsyncEdgeWriter) startBackgroundWriter(ctx context.Context) }() } -func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []any) error { +func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []edge.TraversalInput) error { log.I.Debugf("batch write JanusGraphAsyncEdgeWriter with %d elements", len(data)) defer jge.writingInFligth.Done() - // This seems ~pointless BUT is required to have the ability to use edge.TraversalInput/vertex.TraversalInput - // as the type - // Even tho it's an alias to any, since we use it in an array, we cannot simply .([]any) or vice versa because of the underlying memory layout. - convertedToTraversalInput := make([]edge.TraversalInput, 0) - for _, d := range data { - convertedToTraversalInput = append(convertedToTraversalInput, d.(edge.TraversalInput)) - } - - op := jge.gremlin(jge.traversalSource, convertedToTraversalInput) + op := jge.gremlin(jge.traversalSource, data) promise := op.Iterate() err := <-promise if err != nil { @@ -99,36 +90,32 @@ func (jge *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { return errors.New("JanusGraph traversalSource is not initialized") } - if len(jge.inserts) == 0 { - log.I.Debugf("Skipping flush on edges writer as no write operations left") - // we need to send something to the channel from this function whenever we don't return an error - // we cannot defer it because the go routine may last longer than the current function - // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data - jge.writingInFligth.Wait() - return nil + if len(jge.inserts) != 0 { + jge.writingInFligth.Add(1) + err := jge.batchWrite(ctx, jge.inserts) + if err != nil { + log.I.Errorf("Failed to batch write edge: %+v", err) + jge.writingInFligth.Wait() + return err + } + log.I.Info("Done flushing edges, clearing the queue") + jge.inserts = nil } - jge.writingInFligth.Add(1) - err := jge.batchWrite(ctx, jge.inserts) - if err != nil { - log.I.Errorf("Failed to batch write edge: %+v", err) - jge.writingInFligth.Wait() - return err - } - log.I.Info("Done flushing edges, clearing the queue") - jge.inserts = nil + jge.writingInFligth.Wait() return nil } -func (jge *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, edge any) error { +func (jge *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, e any) error { jge.mu.Lock() defer jge.mu.Unlock() - jge.inserts = append(jge.inserts, edge) + jge.inserts = append(jge.inserts, e) if len(jge.inserts) > jge.batchSize { - copied := make([]any, len(jge.inserts)) + copied := make([]edge.TraversalInput, len(jge.inserts)) copy(copied, jge.inserts) + jge.writingInFligth.Add(1) jge.consumerChan <- copied // cleanup the ops array after we have copied it to the channel jge.inserts = nil diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index a4308e8b..45da4fef 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -51,7 +51,6 @@ func (jgv *JanusGraphAsyncVertexWriter) startBackgroundWriter(ctx context.Contex if data == nil { return } - jgv.writingInFligth.Add(1) err := jgv.batchWrite(ctx, data) if err != nil { log.I.Errorf("failed to write data in background batch writer: %v", err) @@ -68,10 +67,6 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []v log.I.Debugf("batch write JanusGraphAsyncVertexWriter with %d elements", len(data)) defer jgv.writingInFligth.Done() - // convertedToTraversalInput := make([]vertex.TraversalInput, 0) - // for _, d := range data { - // convertedToTraversalInput = append(convertedToTraversalInput, d.(vertex.TraversalInput)) - // } op := jgv.gremlin(jgv.traversalSource, data) promise := op.Iterate() err := <-promise @@ -95,24 +90,19 @@ func (jgv *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { return errors.New("JanusGraph traversalSource is not initialized") } - if len(jgv.inserts) == 0 { - log.I.Debugf("Skipping flush on vertex writer as no write operations left") - // we need to send something to the channel from this function whenever we don't return an error - // we cannot defer it because the go routine may last longer than the current function - // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data - jgv.writingInFligth.Wait() - return nil + if len(jgv.inserts) != 0 { + jgv.writingInFligth.Add(1) + err := jgv.batchWrite(ctx, jgv.inserts) + if err != nil { + log.I.Errorf("Failed to batch write vertex: %+v", err) + jgv.writingInFligth.Wait() + return err + } + log.I.Info("Done flushing vertices, clearing the queue") + jgv.inserts = nil } - jgv.writingInFligth.Add(1) - err := jgv.batchWrite(ctx, jgv.inserts) - if err != nil { - log.I.Errorf("Failed to batch write vertex: %+v", err) - jgv.writingInFligth.Wait() - return err - } - log.I.Info("Done flushing vertices, clearing the queue") - jgv.inserts = nil + jgv.writingInFligth.Wait() return nil } @@ -123,9 +113,9 @@ func (jgv *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, v any) error jgv.inserts = append(jgv.inserts, v) if len(jgv.inserts) > jgv.batchSize { - var copied []vertex.TraversalInput - copied = make([]vertex.TraversalInput, len(jgv.inserts)) + copied := make([]vertex.TraversalInput, len(jgv.inserts)) copy(copied, jgv.inserts) + jgv.writingInFligth.Add(1) jgv.consumerChan <- copied // cleanup the ops array after we have copied it to the channel jgv.inserts = nil From 7a789929aa69361cfb528067b186b6342cb154fd Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 16:54:19 +0200 Subject: [PATCH 48/61] healthcheck --- .../storage/graphdb/janusgraph_provider.go | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 11a3f07c..08488057 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -3,9 +3,11 @@ package graphdb import ( "context" "errors" + "fmt" "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" + "github.com/DataDog/KubeHound/pkg/telemetry/log" gremlingo "github.com/apache/tinkerpop/gremlin-go/driver" ) @@ -38,11 +40,36 @@ func (jgp *JanusGraphProvider) Name() string { return "JanusGraphProvider" } +// HealthCheck sends a single digit, as a string. JanusGraph will reply to this with the same value (arithmetic operation) +// We choose the value "1" because it's not the default int value in case there's an issue somewhere. +// from: https://stackoverflow.com/questions/59396980/gremlin-query-to-check-connection-health func (jgp *JanusGraphProvider) HealthCheck(ctx context.Context) (bool, error) { - if jgp.client != nil { - return true, nil + fmt.Println("health check start") + wantValue := "1" + if jgp.client == nil { + return false, errors.New("failed to get janus graph client (nil)") } - return false, errors.New("failed to get janus graph client") + res, err := jgp.client.Submit(wantValue) + if err != nil { + return false, err + } + + one, ok, err := res.One() + if !ok || err != nil { + return false, fmt.Errorf("failed to get one results from healthcheck, got: %s", one) + } + + value, err := one.GetInt() + if err != nil { + return false, fmt.Errorf("failed to get int value from healthcheck: %v", err) + } + + if value != 1 { + log.I.Errorf("healthcheck returned wrong value, got: %d wanted: %s", value, wantValue) + return false, nil + } + + return true, nil } // Raw returns a handle to the underlying provider to allow implementation specific operations e.g graph queries. From c0b115303a15db39ee40ee488335a14e52b3c29b Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 16:59:54 +0200 Subject: [PATCH 49/61] Add conn timeout flag --- pkg/kubehound/storage/graphdb/janusgraph_provider.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 08488057..853bed1c 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/DataDog/KubeHound/pkg/kubehound/graph/edge" "github.com/DataDog/KubeHound/pkg/kubehound/graph/vertex" @@ -20,11 +21,15 @@ type JanusGraphProvider struct { client *gremlingo.DriverRemoteConnection } -func NewGraphDriver(ctx context.Context, dbHost string) (*JanusGraphProvider, error) { +func NewGraphDriver(ctx context.Context, dbHost string, timeout time.Duration) (*JanusGraphProvider, error) { if dbHost == "" { return nil, errors.New("JanusGraph DB URL is not set") } - driver, err := gremlingo.NewDriverRemoteConnection(dbHost) + driver, err := gremlingo.NewDriverRemoteConnection(dbHost, + func(settings *gremlingo.DriverRemoteConnectionSettings) { + settings.ConnectionTimeout = timeout + }, + ) if err != nil { return nil, err } From b9da2087bec9d6f09903997d1213b7d9508ffdd8 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 17:00:29 +0200 Subject: [PATCH 50/61] missing file --- pkg/kubehound/storage/graphdb/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index de70a688..753e4ca7 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -71,5 +71,5 @@ type AsyncEdgeWriter interface { // Factory returns an initialized instance of a graphdb provider from the provided application config. func Factory(ctx context.Context, cfg *config.KubehoundConfig) (Provider, error) { - return NewGraphDriver(ctx, cfg.JanusGraph.URL) + return NewGraphDriver(ctx, cfg.JanusGraph.URL, cfg.JanusGraph.ConnectionTimeout) } From 81ccaec2eca6ad1d49da5577b96a0e93de46e774 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 17:07:23 +0200 Subject: [PATCH 51/61] typos --- .../storage/graphdb/janusgraph_edge_writer.go | 13 +++++++------ .../storage/graphdb/janusgraph_vertex_writer.go | 13 +++++++------ pkg/kubehound/storage/storedb/mongo_writer.go | 10 +++++----- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index 0f3a5b55..e74820c7 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -17,7 +17,7 @@ type JanusGraphAsyncEdgeWriter struct { traversalSource *gremlingo.GraphTraversalSource inserts []edge.TraversalInput consumerChan chan []edge.TraversalInput - writingInFligth sync.WaitGroup + writingInFlight sync.WaitGroup batchSize int mu sync.Mutex } @@ -66,7 +66,7 @@ func (jge *JanusGraphAsyncEdgeWriter) startBackgroundWriter(ctx context.Context) func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []edge.TraversalInput) error { log.I.Debugf("batch write JanusGraphAsyncEdgeWriter with %d elements", len(data)) - defer jge.writingInFligth.Done() + defer jge.writingInFlight.Done() op := jge.gremlin(jge.traversalSource, data) promise := op.Iterate() @@ -79,6 +79,7 @@ func (jge *JanusGraphAsyncEdgeWriter) batchWrite(ctx context.Context, data []edg } func (jge *JanusGraphAsyncEdgeWriter) Close(ctx context.Context) error { + close(jge.consumerChan) return nil } @@ -91,18 +92,18 @@ func (jge *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { } if len(jge.inserts) != 0 { - jge.writingInFligth.Add(1) + jge.writingInFlight.Add(1) err := jge.batchWrite(ctx, jge.inserts) if err != nil { log.I.Errorf("Failed to batch write edge: %+v", err) - jge.writingInFligth.Wait() + jge.writingInFlight.Wait() return err } log.I.Info("Done flushing edges, clearing the queue") jge.inserts = nil } - jge.writingInFligth.Wait() + jge.writingInFlight.Wait() return nil } @@ -115,7 +116,7 @@ func (jge *JanusGraphAsyncEdgeWriter) Queue(ctx context.Context, e any) error { if len(jge.inserts) > jge.batchSize { copied := make([]edge.TraversalInput, len(jge.inserts)) copy(copied, jge.inserts) - jge.writingInFligth.Add(1) + jge.writingInFlight.Add(1) jge.consumerChan <- copied // cleanup the ops array after we have copied it to the channel jge.inserts = nil diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index 45da4fef..ced0a881 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -17,7 +17,7 @@ type JanusGraphAsyncVertexWriter struct { traversalSource *gremlingo.GraphTraversalSource inserts []vertex.TraversalInput consumerChan chan []vertex.TraversalInput - writingInFligth sync.WaitGroup + writingInFlight sync.WaitGroup batchSize int mu sync.Mutex } @@ -65,7 +65,7 @@ func (jgv *JanusGraphAsyncVertexWriter) startBackgroundWriter(ctx context.Contex func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []vertex.TraversalInput) error { log.I.Debugf("batch write JanusGraphAsyncVertexWriter with %d elements", len(data)) - defer jgv.writingInFligth.Done() + defer jgv.writingInFlight.Done() op := jgv.gremlin(jgv.traversalSource, data) promise := op.Iterate() @@ -77,6 +77,7 @@ func (jgv *JanusGraphAsyncVertexWriter) batchWrite(ctx context.Context, data []v } func (jgv *JanusGraphAsyncVertexWriter) Close(ctx context.Context) error { + close(jgv.consumerChan) return nil } @@ -91,18 +92,18 @@ func (jgv *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { } if len(jgv.inserts) != 0 { - jgv.writingInFligth.Add(1) + jgv.writingInFlight.Add(1) err := jgv.batchWrite(ctx, jgv.inserts) if err != nil { log.I.Errorf("Failed to batch write vertex: %+v", err) - jgv.writingInFligth.Wait() + jgv.writingInFlight.Wait() return err } log.I.Info("Done flushing vertices, clearing the queue") jgv.inserts = nil } - jgv.writingInFligth.Wait() + jgv.writingInFlight.Wait() return nil } @@ -115,7 +116,7 @@ func (jgv *JanusGraphAsyncVertexWriter) Queue(ctx context.Context, v any) error if len(jgv.inserts) > jgv.batchSize { copied := make([]vertex.TraversalInput, len(jgv.inserts)) copy(copied, jgv.inserts) - jgv.writingInFligth.Add(1) + jgv.writingInFlight.Add(1) jgv.consumerChan <- copied // cleanup the ops array after we have copied it to the channel jgv.inserts = nil diff --git a/pkg/kubehound/storage/storedb/mongo_writer.go b/pkg/kubehound/storage/storedb/mongo_writer.go index a235765f..7779fed6 100644 --- a/pkg/kubehound/storage/storedb/mongo_writer.go +++ b/pkg/kubehound/storage/storedb/mongo_writer.go @@ -25,7 +25,7 @@ type MongoAsyncWriter struct { collection *mongo.Collection batchSize int consumerChan chan []mongo.WriteModel - writingInFligth sync.WaitGroup + writingInFlight sync.WaitGroup } func NewMongoAsyncWriter(ctx context.Context, mp *MongoProvider, collection collections.Collection) *MongoAsyncWriter { @@ -63,8 +63,8 @@ func (maw *MongoAsyncWriter) startBackgroundWriter(ctx context.Context) { // batchWrite blocks until the write is complete func (maw *MongoAsyncWriter) batchWrite(ctx context.Context, ops []mongo.WriteModel) error { - maw.writingInFligth.Add(1) - defer maw.writingInFligth.Done() + maw.writingInFlight.Add(1) + defer maw.writingInFlight.Done() bulkWriteOpts := options.BulkWrite().SetOrdered(false) _, err := maw.collection.BulkWrite(ctx, ops, bulkWriteOpts) if err != nil { @@ -100,13 +100,13 @@ func (maw *MongoAsyncWriter) Flush(ctx context.Context) error { // we need to send something to the channel from this function whenever we don't return an error // we cannot defer it because the go routine may last longer than the current function // the defer is going to be executed at the return time, whetever or not the inner go routine is processing data - maw.writingInFligth.Wait() + maw.writingInFlight.Wait() return nil } err := maw.batchWrite(ctx, maw.ops) if err != nil { - maw.writingInFligth.Wait() + maw.writingInFlight.Wait() return err } From fbe47421a507caa167b464ea553b8d791f439784 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 17:07:56 +0200 Subject: [PATCH 52/61] ptrs --- pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go | 2 +- pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index e74820c7..f31a69f2 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -17,7 +17,7 @@ type JanusGraphAsyncEdgeWriter struct { traversalSource *gremlingo.GraphTraversalSource inserts []edge.TraversalInput consumerChan chan []edge.TraversalInput - writingInFlight sync.WaitGroup + writingInFlight *sync.WaitGroup batchSize int mu sync.Mutex } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index ced0a881..c942b43e 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -17,7 +17,7 @@ type JanusGraphAsyncVertexWriter struct { traversalSource *gremlingo.GraphTraversalSource inserts []vertex.TraversalInput consumerChan chan []vertex.TraversalInput - writingInFlight sync.WaitGroup + writingInFlight *sync.WaitGroup batchSize int mu sync.Mutex } From baad7ad711e3f52714cc49ca346b6aa2c4437a4e Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 17:10:48 +0200 Subject: [PATCH 53/61] oops --- pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go | 2 +- pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index f31a69f2..a79a298c 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -30,12 +30,12 @@ func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemo } source := gremlingo.Traversal_().WithRemote(drc) - jw := JanusGraphAsyncEdgeWriter{ gremlin: e.Traversal(), inserts: make([]edge.TraversalInput, 0, e.BatchSize()), traversalSource: source, batchSize: e.BatchSize(), + writingInFlight: &sync.WaitGroup{}, consumerChan: make(chan []edge.TraversalInput, e.BatchSize()*channelSizeBatchFactor), } jw.startBackgroundWriter(ctx) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index c942b43e..ab3f1344 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -35,6 +35,7 @@ func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRe inserts: make([]vertex.TraversalInput, 0, v.BatchSize()), traversalSource: source, batchSize: v.BatchSize(), + writingInFlight: &sync.WaitGroup{}, consumerChan: make(chan []vertex.TraversalInput, v.BatchSize()*channelSizeBatchFactor), } jw.startBackgroundWriter(ctx) From 12d2a3710f06f5debaeb4366a1ba4645aac9ff85 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 17:12:23 +0200 Subject: [PATCH 54/61] remve container attach --- pkg/kubehound/graph/edge/container_attach.go | 98 -------------------- 1 file changed, 98 deletions(-) delete mode 100644 pkg/kubehound/graph/edge/container_attach.go diff --git a/pkg/kubehound/graph/edge/container_attach.go b/pkg/kubehound/graph/edge/container_attach.go deleted file mode 100644 index 22c02237..00000000 --- a/pkg/kubehound/graph/edge/container_attach.go +++ /dev/null @@ -1,98 +0,0 @@ -package edge - -import ( - "context" - - "github.com/DataDog/KubeHound/pkg/kubehound/storage/storedb" - "github.com/DataDog/KubeHound/pkg/kubehound/store/collections" - "github.com/DataDog/KubeHound/pkg/telemetry/log" - gremlin "github.com/apache/tinkerpop/gremlin-go/driver" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func init() { - Register(ContainerAttach{}) -} - -// @@DOCLINK: https://datadoghq.atlassian.net/wiki/spaces/ASE/pages/2883354625/CONTAINER+ATTACH -type ContainerAttach struct { -} - -type ContainerAttachGroup struct { - Pod primitive.ObjectID `bson:"_id" json:"pod"` - Containers []primitive.ObjectID `bson:"containers" json:"containers"` -} - -func (e ContainerAttach) Label() string { - return "CONTAINER_ATTACH" -} - -func (e ContainerAttach) BatchSize() int { - return DefaultBatchSize -} - -func (e ContainerAttach) Traversal() EdgeTraversal { - return func(source *gremlin.GraphTraversalSource, inserts []TraversalInput) *gremlin.GraphTraversal { - log.I.Errorf("%s edge traversal does not work, inserting dummy input", e.Label()) - - g := source.GetGraphTraversal() - // return g.Inject(inserts).Unfold().As("ca"). - // V().HasLabel("Pod").Has("storeID", gremlin.T__.Select("ca").Select("pod")).As("pod"). - // V().HasLabel("Container").Has("storeID", gremlin.T__.Select("ca").Select("container")).As("container"). - // MergeE(e.Label()).From("pod").To("container") - - g.AddV("dummy_entry") - - // for _, i := range inserts { - // data := i.(*ContainerAttachGroup) - // podId, err := data.Pod.MarshalJSON() - // if err != nil { - // log.I.Errorf("failed to get pod id: %v", err) - // } - - // for _, container := range data.Containers { - // containerID, err := container.MarshalJSON() - // if err != nil { - // log.I.Errorf("failed to get pod id: %v", err) - // } - // pod, err := g.V("Pod").Has("store_id", podId).Next() - // containers, err := g.V("Container").Has("store_id", containerID).Next() - // clist, err := containers.GetSlice() - - // for _, container := range *clist { - // fmt.Printf("containers edge blabla %+v\n", container) - // g = g.V(pod).AddE(e.Label()).To(container) - // } - // } - // } - return g - } -} - -func (e ContainerAttach) Processor(ctx context.Context, entry DataContainer) (TraversalInput, error) { - return MongoProcessor[*ContainerAttachGroup](ctx, entry) -} - -func (e ContainerAttach) Stream(ctx context.Context, store storedb.Provider, - callback ProcessEntryCallback, complete CompleteQueryCallback) error { - - containers := MongoDB(store).Collection(collections.ContainerName) - pipeline := []bson.M{ - {"$group": bson.M{ - "_id": "$pod_id", - "containers": bson.M{ - "$push": "$_id", - }, - }, - }, - } - - cur, err := containers.Aggregate(context.Background(), pipeline) - if err != nil { - return err - } - defer cur.Close(ctx) - - return MongoCursorHandler[ContainerAttachGroup](ctx, cur, callback, complete) -} From 8da0fb46ebc2f87abea126a96e3e109ac082cf5f Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 17:15:17 +0200 Subject: [PATCH 55/61] Bump default batch size to 50 :shrug: --- pkg/kubehound/graph/vertex/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubehound/graph/vertex/builder.go b/pkg/kubehound/graph/vertex/builder.go index f4f70701..1cc61e09 100644 --- a/pkg/kubehound/graph/vertex/builder.go +++ b/pkg/kubehound/graph/vertex/builder.go @@ -5,7 +5,7 @@ import ( ) const ( - DefaultBatchSize = 5 + DefaultBatchSize = 50 ) // An object to be consumed by a vertex traversal function to insert a vertex into the graph database. From c78a139e555b08d0035052c39ab5202706dd13ad Mon Sep 17 00:00:00 2001 From: jt-dd <112463504+jt-dd@users.noreply.github.com> Date: Mon, 12 Jun 2023 18:09:22 +0200 Subject: [PATCH 56/61] Default page size typo (#41) --- pkg/globals/collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/globals/collector.go b/pkg/globals/collector.go index e6560635..74b9328c 100644 --- a/pkg/globals/collector.go +++ b/pkg/globals/collector.go @@ -1,7 +1,7 @@ package globals const ( - DefaultK8sAPIPageSize int64 = 5000 + DefaultK8sAPIPageSize int64 = 500 DefaultK8sAPIPageBufferSize int32 = 10 DefaultK8sAPIRateLimitPerSecond int = 100 ) From 621c1a6cc50a51ce859ec5f321fc2af248a8d271 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 20:05:14 +0200 Subject: [PATCH 57/61] fix pr comments --- pkg/kubehound/ingestor/pipeline/ingest_resources.go | 4 +--- .../storage/graphdb/janusgraph_edge_writer.go | 1 - pkg/kubehound/storage/graphdb/janusgraph_provider.go | 12 ++---------- .../storage/graphdb/janusgraph_vertex_writer.go | 1 - pkg/kubehound/storage/graphdb/provider.go | 8 -------- 5 files changed, 3 insertions(+), 23 deletions(-) diff --git a/pkg/kubehound/ingestor/pipeline/ingest_resources.go b/pkg/kubehound/ingestor/pipeline/ingest_resources.go index 4ec31ffb..17ff84b9 100644 --- a/pkg/kubehound/ingestor/pipeline/ingest_resources.go +++ b/pkg/kubehound/ingestor/pipeline/ingest_resources.go @@ -86,9 +86,7 @@ func WithStoreWriter[T collections.Collection](c T) IngestResourceOption { // To access the writer use the graphWriter(v vertex.Vertex) function. func WithGraphWriter(v vertex.Builder) IngestResourceOption { return func(ctx context.Context, rOpts *resourceOptions, deps *Dependencies) error { - opts := []graphdb.WriterOption{ - // graphdb.WithTransaction(), // disable options for now - } + opts := []graphdb.WriterOption{} w, err := deps.GraphDB.VertexWriter(ctx, v, opts...) if err != nil { diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index a79a298c..00b01fa8 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -23,7 +23,6 @@ type JanusGraphAsyncEdgeWriter struct { } func NewJanusGraphAsyncEdgeWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, e edge.Builder, opts ...WriterOption) (*JanusGraphAsyncEdgeWriter, error) { - log.I.Infof("Created new JanusGraphAsyncEdgeWriter") options := &writerOptions{} for _, opt := range opts { opt(options) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index 853bed1c..d715b266 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -84,20 +84,12 @@ func (jgp *JanusGraphProvider) Raw() any { // VertexWriter creates a new AsyncVertexWriter instance to enable asynchronous bulk inserts of vertices. func (jgp *JanusGraphProvider) VertexWriter(ctx context.Context, v vertex.Builder, opts ...WriterOption) (AsyncVertexWriter, error) { - writer, err := NewJanusGraphAsyncVertexWriter(ctx, jgp.client, v, opts...) - if err != nil { - return nil, err - } - return writer, nil + return NewJanusGraphAsyncVertexWriter(ctx, jgp.client, v, opts...) } // EdgeWriter creates a new AsyncEdgeWriter instance to enable asynchronous bulk inserts of edges. func (jgp *JanusGraphProvider) EdgeWriter(ctx context.Context, e edge.Builder, opts ...WriterOption) (AsyncEdgeWriter, error) { - writer, err := NewJanusGraphAsyncEdgeWriter(ctx, jgp.client, e, opts...) - if err != nil { - return nil, err - } - return writer, nil + return NewJanusGraphAsyncEdgeWriter(ctx, jgp.client, e, opts...) } // Close cleans up any resources used by the Provider implementation. Provider cannot be reused after this call. diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index ab3f1344..bd78412a 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -23,7 +23,6 @@ type JanusGraphAsyncVertexWriter struct { } func NewJanusGraphAsyncVertexWriter(ctx context.Context, drc *gremlingo.DriverRemoteConnection, v vertex.Builder, opts ...WriterOption) (*JanusGraphAsyncVertexWriter, error) { - log.I.Infof("Created new JanusGraphAsyncVertexWriter") options := &writerOptions{} for _, opt := range opts { opt(options) diff --git a/pkg/kubehound/storage/graphdb/provider.go b/pkg/kubehound/storage/graphdb/provider.go index 753e4ca7..18e20745 100644 --- a/pkg/kubehound/storage/graphdb/provider.go +++ b/pkg/kubehound/storage/graphdb/provider.go @@ -10,18 +10,10 @@ import ( ) type writerOptions struct { - isTransactionEnabled bool } type WriterOption func(*writerOptions) -// WithConfigPath sets the path for the KubeHound config file. -func WithTransaction() WriterOption { - return func(opt *writerOptions) { - opt.isTransactionEnabled = true - } -} - // Provider defines the interface for implementations of the graphdb provider for storage of the calculated K8s attack graph. // //go:generate mockery --name Provider --output mocks --case underscore --filename graph_provider.go --with-expecter From 2933401c2dc3363e47facf16c56b51d8c947baeb Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 20:11:03 +0200 Subject: [PATCH 58/61] fix pr comments --- pkg/kubehound/storage/graphdb/janusgraph_provider.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/kubehound/storage/graphdb/janusgraph_provider.go b/pkg/kubehound/storage/graphdb/janusgraph_provider.go index d715b266..4b2feded 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_provider.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_provider.go @@ -49,10 +49,9 @@ func (jgp *JanusGraphProvider) Name() string { // We choose the value "1" because it's not the default int value in case there's an issue somewhere. // from: https://stackoverflow.com/questions/59396980/gremlin-query-to-check-connection-health func (jgp *JanusGraphProvider) HealthCheck(ctx context.Context) (bool, error) { - fmt.Println("health check start") wantValue := "1" if jgp.client == nil { - return false, errors.New("failed to get janus graph client (nil)") + return false, errors.New("get janus graph client (nil)") } res, err := jgp.client.Submit(wantValue) if err != nil { @@ -61,12 +60,12 @@ func (jgp *JanusGraphProvider) HealthCheck(ctx context.Context) (bool, error) { one, ok, err := res.One() if !ok || err != nil { - return false, fmt.Errorf("failed to get one results from healthcheck, got: %s", one) + return false, fmt.Errorf("get one results from healthcheck, got: %s", one) } value, err := one.GetInt() if err != nil { - return false, fmt.Errorf("failed to get int value from healthcheck: %v", err) + return false, fmt.Errorf("get int value from healthcheck: %v", err) } if value != 1 { From d59705c9eecbdf3c18bae11879cf46872cbce3ee Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 20:13:19 +0200 Subject: [PATCH 59/61] replace failed to by error level logs --- pkg/kubehound/ingestor/pipeline/pod_ingest.go | 2 +- pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go | 4 ++-- pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go | 4 ++-- pkg/kubehound/storage/storedb/mongo_writer.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/kubehound/ingestor/pipeline/pod_ingest.go b/pkg/kubehound/ingestor/pipeline/pod_ingest.go index dd8aa6e8..4e335aaf 100644 --- a/pkg/kubehound/ingestor/pipeline/pod_ingest.go +++ b/pkg/kubehound/ingestor/pipeline/pod_ingest.go @@ -109,7 +109,7 @@ func (i *PodIngest) processVolume(ctx context.Context, parent *store.Pod, volume // Normalize volume to store object format sv, err := i.r.storeConvert.Volume(ctx, volume, parent) if err != nil { - log.I.Errorf("failed to process volume type: %v (continuing)", err) + log.I.Errorf("process volume type: %v (continuing)", err) return nil } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go index 00b01fa8..ab0f36fe 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_edge_writer.go @@ -53,7 +53,7 @@ func (jge *JanusGraphAsyncEdgeWriter) startBackgroundWriter(ctx context.Context) } err := jge.batchWrite(ctx, data) if err != nil { - log.I.Errorf("failed to write data in background batch writer: %v", err) + log.I.Errorf("write data in background batch writer: %v", err) } case <-ctx.Done(): log.I.Info("Closed background janusgraph worker") @@ -94,7 +94,7 @@ func (jge *JanusGraphAsyncEdgeWriter) Flush(ctx context.Context) error { jge.writingInFlight.Add(1) err := jge.batchWrite(ctx, jge.inserts) if err != nil { - log.I.Errorf("Failed to batch write edge: %+v", err) + log.I.Errorf("batch write edge: %+v", err) jge.writingInFlight.Wait() return err } diff --git a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go index bd78412a..bb3a8f5d 100644 --- a/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go +++ b/pkg/kubehound/storage/graphdb/janusgraph_vertex_writer.go @@ -53,7 +53,7 @@ func (jgv *JanusGraphAsyncVertexWriter) startBackgroundWriter(ctx context.Contex } err := jgv.batchWrite(ctx, data) if err != nil { - log.I.Errorf("failed to write data in background batch writer: %v", err) + log.I.Errorf("write data in background batch writer: %v", err) } case <-ctx.Done(): log.I.Info("Closed background janusgraph worker") @@ -95,7 +95,7 @@ func (jgv *JanusGraphAsyncVertexWriter) Flush(ctx context.Context) error { jgv.writingInFlight.Add(1) err := jgv.batchWrite(ctx, jgv.inserts) if err != nil { - log.I.Errorf("Failed to batch write vertex: %+v", err) + log.I.Errorf("batch write vertex: %+v", err) jgv.writingInFlight.Wait() return err } diff --git a/pkg/kubehound/storage/storedb/mongo_writer.go b/pkg/kubehound/storage/storedb/mongo_writer.go index 7779fed6..c3c86e47 100644 --- a/pkg/kubehound/storage/storedb/mongo_writer.go +++ b/pkg/kubehound/storage/storedb/mongo_writer.go @@ -51,7 +51,7 @@ func (maw *MongoAsyncWriter) startBackgroundWriter(ctx context.Context) { } err := maw.batchWrite(ctx, data) if err != nil { - log.I.Errorf("failed to write data in background batch writer: %v", err) + log.I.Errorf("write data in background batch writer: %v", err) } case <-ctx.Done(): log.I.Info("Closed background mongodb worker") From b62f280f68751d0223073fd8329fef9b3697dc64 Mon Sep 17 00:00:00 2001 From: jt-dd Date: Mon, 12 Jun 2023 20:14:53 +0200 Subject: [PATCH 60/61] merged too fast --- configs/etc/kubehound.yaml | 2 +- pkg/config/config_test.go | 4 ++-- pkg/config/testdata/kubehound-k8s-collector.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/etc/kubehound.yaml b/configs/etc/kubehound.yaml index ebc6f3ff..6b6fe6ef 100644 --- a/configs/etc/kubehound.yaml +++ b/configs/etc/kubehound.yaml @@ -1,7 +1,7 @@ collector: type: live-k8s-api-collector live: - page_size: 5000 + page_size: 500 page_buffer_size: 10 rate_limit_per_second: 100 mongodb: diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index adf88ea8..d5aed35f 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,7 +30,7 @@ func TestMustLoadConfig(t *testing.T) { }, // This is always set as the default value Live: &K8SAPICollectorConfig{ - PageSize: 5000, + PageSize: 500, PageBufferSize: 10, RateLimitPerSecond: 100, }, @@ -55,7 +55,7 @@ func TestMustLoadConfig(t *testing.T) { Collector: CollectorConfig{ Type: CollectorTypeK8sAPI, Live: &K8SAPICollectorConfig{ - PageSize: 5000, + PageSize: 500, PageBufferSize: 10, RateLimitPerSecond: 100, }, diff --git a/pkg/config/testdata/kubehound-k8s-collector.yaml b/pkg/config/testdata/kubehound-k8s-collector.yaml index ebc6f3ff..6b6fe6ef 100644 --- a/pkg/config/testdata/kubehound-k8s-collector.yaml +++ b/pkg/config/testdata/kubehound-k8s-collector.yaml @@ -1,7 +1,7 @@ collector: type: live-k8s-api-collector live: - page_size: 5000 + page_size: 500 page_buffer_size: 10 rate_limit_per_second: 100 mongodb: From 1f602960ddef8d9ddbd85139b952f3977037e690 Mon Sep 17 00:00:00 2001 From: Edouard Schweisguth Date: Mon, 12 Jun 2023 20:21:28 +0200 Subject: [PATCH 61/61] fix config laod --- pkg/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index adf88ea8..fe828660 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,7 +30,7 @@ func TestMustLoadConfig(t *testing.T) { }, // This is always set as the default value Live: &K8SAPICollectorConfig{ - PageSize: 5000, + PageSize: 500, PageBufferSize: 10, RateLimitPerSecond: 100, },