guac all-in-one cmdline

Makefile

@@ -37,6 +37,8 @@ ci: fmt lint test ## Run all the tests and code checks
 .PHONY: build
 build: ## Build a version
 	go build -ldflags ${LDFLAGS} -o bin/collector cmd/collector/main.go
+	go build -ldflags ${LDFLAGS} -o bin/ingest cmd/ingest/main.go
+	go build -ldflags ${LDFLAGS} -o bin/guacone cmd/guacone/main.go
 
 .PHONY: clean
 clean: ## Remove temporary files

cmd/guacone/cmd/files.go

@@ -0,0 +1,202 @@
+//
+// Copyright 2022 The GUAC Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/guacsec/guac/pkg/assembler"
+	"github.com/guacsec/guac/pkg/assembler/graphdb"
+	"github.com/guacsec/guac/pkg/handler/collector"
+	"github.com/guacsec/guac/pkg/handler/collector/file"
+	"github.com/guacsec/guac/pkg/handler/processor"
+	"github.com/guacsec/guac/pkg/handler/processor/process"
+	"github.com/guacsec/guac/pkg/ingestor/parser"
+	"github.com/guacsec/guac/pkg/logging"
+	"github.com/spf13/cobra"
+)
+
+var flags = struct {
+	dbAddr string
+	creds  string
+	realm  string
+}{}
+
+type options struct {
+	dbAddr string
+	user   string
+	pass   string
+	realm  string
+
+	// path to folder with documents to collect
+	path string
+}
+
+func init() {
+	exampleCmd.PersistentFlags().StringVar(&flags.dbAddr, "db-addr", "neo4j://localhost:7687", "address to neo4j db")
+	exampleCmd.PersistentFlags().StringVar(&flags.creds, "creds", "", "credentials to access neo4j in 'user:pass' format")
+	exampleCmd.PersistentFlags().StringVar(&flags.realm, "realm", "neo4j", "realm to connecto graph db")
+	_ = exampleCmd.MarkPersistentFlagRequired("creds")
+}
+
+var exampleCmd = &cobra.Command{
+	Use:   "files [flags] file_path",
+	Short: "take a folder of files and create a GUAC graph",
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := logging.WithLogger(context.Background())
+		logger := logging.FromContext(ctx)
+
+		opts, err := validateFlags(args)
+		if err != nil {
+			fmt.Printf("unable to validate flags: %v\n", err)
+			_ = cmd.Help()
+			os.Exit(1)
+		}
+
+		// Register collector
+		fileCollector := file.NewFileCollector(ctx, opts.path, false, time.Second)
+		err = collector.RegisterDocumentCollector(fileCollector, file.FileCollector)
+		if err != nil {
+			logger.Errorf("unable to register file collector: %v", err)
+		}
+
+		// Get pipeline of components
+		processorFunc, err := getProcessor(ctx)
+		if err != nil {
+			logger.Errorf("error: %v", err)
+			os.Exit(1)
+		}
+		ingestorFunc, err := getIngestor()
+		if err != nil {
+			logger.Errorf("error: %v", err)
+			os.Exit(1)
+		}
+		assemblerFunc, err := getAssembler(opts)
+		if err != nil {
+			logger.Errorf("error: %v", err)
+			os.Exit(1)
+		}
+
+		// Set emit function to go through the entire pipeline
+		emit := func(d *processor.Document) {
+			docTree, err := processorFunc(d)
+			if err != nil {
+				logger.Errorf("unable to process doc: %v, fomat: %v, document: %v", err, d.Format, d.Type)
+				return
+			}
+
+			graphs, err := ingestorFunc(docTree)
+			if err != nil {
+				logger.Errorf("unable to ingest doc tree: %v", err)
+				return
+			}
+
+			err = assemblerFunc(graphs)
+			if err != nil {
+				logger.Errorf("unable to assemble graphs: %v", err)
+				return
+			}
+		}
+
+		// Collect
+		docChan, errChan, numCollectors, err := collector.Collect(ctx)
+		if err != nil {
+			logger.Fatal(err)
+		}
+
+		collectorsDone := 0
+		for collectorsDone < numCollectors {
+			select {
+			case d := <-docChan:
+				logger.Infof("emitting doc: %v, fomat: %v, document: %v", string(d.Blob[:10]), d.Format, d.Type)
+				emit(d)
+			case err = <-errChan:
+				if err != nil {
+					logger.Errorf("collector ended with error: %v", err)
+				} else {
+					logger.Info("collector ended gracefully")
+				}
+				collectorsDone += 1
+			}
+		}
+
+		// Drain anything left in document channel
+		for len(docChan) > 0 {
+			d := <-docChan
+			logger.Infof("emitting doc: %v, fomat: %v, document: %v", string(d.Blob[:10]), d.Format, d.Type)
+			emit(d)
+		}
+	},
+}
+
+func validateFlags(args []string) (options, error) {
+	var opts options
+	credsSplit := strings.Split(flags.creds, ":")
+	if len(credsSplit) != 2 {
+		return opts, fmt.Errorf("creds flag not in correct format user:pass")
+	}
+	opts.user = credsSplit[0]
+	opts.pass = credsSplit[1]
+	opts.dbAddr = flags.dbAddr
+
+	if len(args) != 1 {
+		return opts, fmt.Errorf("expected positional argument for file_path")
+	}
+	opts.path = args[0]
+
+	return opts, nil
+}
+
+func getProcessor(ctx context.Context) (func(*processor.Document) (processor.DocumentTree, error), error) {
+	return func(d *processor.Document) (processor.DocumentTree, error) {
+		return process.Process(ctx, d)
+	}, nil
+}
+func getIngestor() (func(processor.DocumentTree) ([]assembler.Graph, error), error) {
+	return func(doc processor.DocumentTree) ([]assembler.Graph, error) {
+		inputs, err := parser.ParseDocumentTree(doc)
+		if err != nil {
+			return nil, err
+		}
+		return inputs, nil
+	}, nil
+}
+
+func getAssembler(opts options) (func([]assembler.Graph) error, error) {
+	authToken := graphdb.CreateAuthTokenWithUsernameAndPassword(opts.user, opts.pass, opts.realm)
+	client, err := graphdb.NewGraphClient(opts.dbAddr, authToken)
+	if err != nil {
+		return nil, err
+	}
+	return func(gs []assembler.Graph) error {
+		combined := assembler.Graph{
+			Nodes: []assembler.GuacNode{},
+			Edges: []assembler.GuacEdge{},
+		}
+		for _, g := range gs {
+			combined.AppendGraph(g)
+		}
+		if err := assembler.StoreGraph(combined, client); err != nil {
+			return err
+		}
+
+		return nil
+	}, nil
+}

cmd/guacone/cmd/root.go

@@ -0,0 +1,39 @@
+//
+// Copyright 2022 The GUAC Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(exampleCmd)
+}
+
+var rootCmd = &cobra.Command{
+	Use:   "guacone",
+	Short: "guacone is an all in one flow cmdline for GUAC",
+}
+
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

cmd/guacone/main.go

@@ -0,0 +1,24 @@
+//
+// Copyright 2022 The GUAC Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/guacsec/guac/cmd/guacone/cmd"
+)
+
+func main() {
+	cmd.Execute()
+}

pkg/handler/processor/ite6/ite6.go

@@ -28,8 +28,8 @@ type ITE6Processor struct {
 
 // ValidateSchema ensures that the document blob can be parsed into a valid data structure
 func (e *ITE6Processor) ValidateSchema(i *processor.Document) error {
-	if i.Type != processor.DocumentITE6Unknown {
-		return fmt.Errorf("expected document type: %v, actual document type: %v", processor.DocumentITE6Unknown, i.Type)
+	if i.Type != processor.DocumentITE6Unknown && i.Type != processor.DocumentITE6SLSA {
+		return fmt.Errorf("expected ITE6 document type, actual document type: %v", i.Type)
 	}
 
 	_, err := parseStatement(i.Blob)

pkg/handler/processor/process/process.go

@@ -21,15 +21,19 @@ import (
 	"fmt"
 
 	"github.com/guacsec/guac/pkg/handler/processor"
+	"github.com/guacsec/guac/pkg/handler/processor/dsse"
 	"github.com/guacsec/guac/pkg/handler/processor/guesser"
+	"github.com/guacsec/guac/pkg/handler/processor/ite6"
 )
 
 var (
 	documentProcessors = map[processor.DocumentType]processor.DocumentProcessor{}
 )
 
 func init() {
-	// Registerprocessor.DocumentProcessor()
+	_ = RegisterDocumentProcessor(&ite6.ITE6Processor{}, processor.DocumentITE6Unknown)
+	_ = RegisterDocumentProcessor(&ite6.ITE6Processor{}, processor.DocumentITE6SLSA)
+	_ = RegisterDocumentProcessor(&dsse.DSSEProcessor{}, processor.DocumentDSSE)
 }
 
 func RegisterDocumentProcessor(p processor.DocumentProcessor, d processor.DocumentType) error {