add file storage

.gitignore

@@ -17,3 +17,10 @@
 *.json
 .gitignore
 .idea/
+
+## protect against development environment being leaked out
+*.pem
+*.nix
+archivist
+archivistctl
+.witness.yaml

Dockerfile

@@ -8,4 +8,5 @@ RUN --mount=target=. --mount=target=/root/.cache,type=cache \
 
 FROM alpine
 COPY --from=build /out/archivist /bin/archivist
+RUN mkdir /tmp/archivist
 ENTRYPOINT ["/bin/archivist"]

cmd/archivist/main.go

@@ -26,6 +26,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/testifysec/archivist/internal/storage/filestore"
+
 	"github.com/testifysec/archivist-api/pkg/api/archivist"
 	"github.com/testifysec/archivist/internal/config"
 	"github.com/testifysec/archivist/internal/server"
@@ -111,13 +113,16 @@ func main() {
 		log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 2: SKIPPED")
 	}
 	// ********************************************************************************
-	log.FromContext(ctx).Infof("executing phase 3: initializing badger (time since start: %s)", time.Since(startTime))
+	log.FromContext(ctx).Infof("executing phase 3: initializing storage clients (time since start: %s)", time.Since(startTime))
 	// ********************************************************************************
 	now = time.Now()
 
-	store, storeCh, err := mysqlstore.NewServer(ctx, "")
+	// TODO make the fileserver optional
+	fileStore, fileStoreCh, err := filestore.NewServer(ctx, cfg.FileDir, cfg.FileServeOn)
+
+	mysqlStore, mysqlStoreCh, err := mysqlstore.NewServer(ctx, "", fileStore)
 	if err != nil {
-		logrus.Fatalf("error starting badger store: %+v", err)
+		logrus.Fatalf("error starting badger mysqlStore: %+v", err)
 	}
 
 	log.FromContext(ctx).WithField("duration", time.Since(now)).Infof("completed phase 3: initializing badger")
@@ -132,10 +137,10 @@ func main() {
 	}
 
 	grpcServer := grpc.NewServer(grpcOptions...)
-	archivistService := server.NewArchivistServer(store)
+	archivistService := server.NewArchivistServer(mysqlStore)
 	archivist.RegisterArchivistServer(grpcServer, archivistService)
 
-	collectorService := server.NewCollectorServer(store)
+	collectorService := server.NewCollectorServer(mysqlStore)
 	archivist.RegisterCollectorServer(grpcServer, collectorService)
 
 	srvErrCh := grpcutils.ListenAndServe(ctx, &cfg.ListenOn, grpcServer)
@@ -147,7 +152,8 @@ func main() {
 
 	<-ctx.Done()
 	<-srvErrCh
-	<-storeCh
+	<-fileStoreCh
+	<-mysqlStoreCh
 
 	//// ********************************************************************************
 	log.FromContext(ctx).Infof("exiting, uptime: %v", time.Since(startTime))

compose.yml

@@ -2,7 +2,7 @@ version: '3.9'
 
 services:
   db:
-    image: mysql
+    image: mysql:oracle
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     ports:
@@ -17,5 +17,8 @@ services:
     environment:
       ARCHIVIST_ENABLE_SPIFFE: "false"
       ARCHIVIST_LISTEN_ON: tcp://0.0.0.0:8080
+      ARCHIVIST_FILE_DIR: /tmp/archivist/
+      ARCHIVIST_FILE_SERVE_ON: :8081
     ports:
       - 8080:8080
+      - 8081:8081

go.mod

@@ -8,11 +8,12 @@ require (
 	github.com/antonfisher/nested-logrus-formatter v1.3.1
 	github.com/git-bom/gitbom-go v0.0.0-20220502033008-4a48bb2317f7
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/gorilla/handlers v1.5.1
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/networkservicemesh/sdk v1.3.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spiffe/go-spiffe/v2 v2.0.0
-	github.com/testifysec/archivist-api v0.0.0-20220502044345-059a521f0ad1
+	github.com/testifysec/archivist-api v0.0.0-20220603060816-12e25ceccbf1
 	github.com/testifysec/witness v0.1.7
 	google.golang.org/grpc v1.46.0
 	google.golang.org/protobuf v1.28.0
@@ -23,6 +24,7 @@ require (
 	github.com/agext/levenshtein v1.2.1 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
+	github.com/felixge/httpsnoop v1.0.2 // indirect
 	github.com/go-logr/logr v1.2.1 // indirect
 	github.com/go-logr/stdr v1.2.0 // indirect
 	github.com/go-openapi/inflect v0.19.0 // indirect

go.sum

@@ -64,6 +64,9 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
+github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -129,6 +132,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
+github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v0.0.0-20181024020800-521ea7b17d02/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
@@ -180,7 +185,6 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
@@ -202,7 +206,6 @@ github.com/networkservicemesh/sdk v1.3.0/go.mod h1:lThahTYTHh1JWlAPrtfmVsRLK7vFJ
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
@@ -233,7 +236,6 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@@ -244,10 +246,8 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cobra v0.0.0-20181021141114-fe5e611709b0/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0=
 github.com/spf13/pflag v0.0.0-20181024212040-082b515c9490/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spiffe/go-spiffe/v2 v2.0.0 h1:y6N7BZAxgaFZYELyrIdxSMm2e2tWpzgQewUts9h1hfM=
 github.com/spiffe/go-spiffe/v2 v2.0.0/go.mod h1:TEfgrEcyFhuSuvqohJt6IxENUNeHfndWCCV1EX7UaVk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -259,8 +259,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
-github.com/testifysec/archivist-api v0.0.0-20220502044345-059a521f0ad1 h1:vHF1EoKlc2eTFsXaHH6fKgib+YMHWi890YpTqH+sYEw=
-github.com/testifysec/archivist-api v0.0.0-20220502044345-059a521f0ad1/go.mod h1:HWpNFd8qFXCoU8gEF/xiuG10ni9EBFhPcpAFTcWDAmc=
+github.com/testifysec/archivist-api v0.0.0-20220603060816-12e25ceccbf1 h1:I1hggIy2JI6+jJ+gqIY3xyl77XBJeFfjXJMMCQ4MIq8=
+github.com/testifysec/archivist-api v0.0.0-20220603060816-12e25ceccbf1/go.mod h1:HWpNFd8qFXCoU8gEF/xiuG10ni9EBFhPcpAFTcWDAmc=
 github.com/testifysec/witness v0.1.7 h1:5qX6Iibm2rSbW1IB9tNvfOaxUMvOB/WNEBl+k5W8wxQ=
 github.com/testifysec/witness v0.1.7/go.mod h1:DfVd6Y8FG3buup5X/d0vZ4ZjcOzK8X6cayaLvDx7e08=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -436,7 +436,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.9-0.20211216111533-8d383106f7e7 h1:M1gcVrIb2lSn2FIL19DG0+/b8nNVKJ7W7b4WcAGZAYM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/config/config.go

@@ -15,14 +15,18 @@
 package config
 
 import (
-	"github.com/kelseyhightower/envconfig"
 	"net/url"
+
+	"github.com/kelseyhightower/envconfig"
 )
 
 type Config struct {
 	EnableSPIFFE bool    `default:"TRUE" desc:"Enable SPIFFE support" split_words:"true"`
 	ListenOn     url.URL `default:"unix:///listen.on.socket" desc:"url to listen on" split_words:"true"`
 	LogLevel     string  `default:"INFO" desc:"Log level" split_words:"true"`
+
+	FileServeOn string `default:"" desc:"What address to serve files on, leave empty to shut off" split_words:"true"`
+	FileDir     string `default:"/tmp/archivist/" desc:"Directory to store and serve files" split_words:"true"`
 }
 
 // Process reads config from env

internal/storage/filestore/file.go

@@ -0,0 +1,72 @@
+package filestore
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"path/filepath"
+
+	"github.com/git-bom/gitbom-go"
+	"github.com/gorilla/handlers"
+	"github.com/sirupsen/logrus"
+	"github.com/testifysec/archivist-api/pkg/api/archivist"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+type UnifiedStorage interface {
+	archivist.CollectorServer
+	archivist.StorageServer
+}
+
+type store struct {
+	archivist.UnimplementedCollectorServer
+	archivist.UnimplementedStorageServer
+
+	prefix string
+}
+
+func NewServer(ctx context.Context, directory string, address string) (UnifiedStorage, <-chan error, error) {
+	errCh := make(chan error)
+	go func() {
+
+		server := handlers.CompressHandler(http.FileServer(http.Dir(directory)))
+		log.Fatalln(http.ListenAndServe(address, server))
+
+		<-ctx.Done()
+		close(errCh)
+	}()
+
+	return &store{
+		prefix: directory,
+	}, errCh, nil
+}
+
+func (s *store) Get(ctx context.Context, request *archivist.GetRequest) (*archivist.GetResponse, error) {
+	res, err := ioutil.ReadFile(filepath.Join(s.prefix, request.GetGitoid()))
+	if err != nil {
+		logrus.WithContext(ctx).Errorf("failed to retrieve object", err)
+		return nil, err
+	}
+	return &archivist.GetResponse{
+		Gitoid: request.GetGitoid(),
+		Object: string(res),
+	}, nil
+}
+
+func (s *store) Store(ctx context.Context, request *archivist.StoreRequest) (*emptypb.Empty, error) {
+	// TODO refactor this to use common code
+	gb := gitbom.NewSha256GitBom()
+	if err := gb.AddReference([]byte(request.Object), nil); err != nil {
+		logrus.WithContext(ctx).Errorf("gitbom tag generation failed: %+v", err)
+		return nil, err
+	}
+
+	fmt.Printf("Writing: %s/%s\n", s.prefix, gb.Identity())
+	err := ioutil.WriteFile(filepath.Join(s.prefix, gb.Identity()+".json"), []byte(request.Object), 0644)
+	if err != nil {
+		return nil, err
+	}
+	return &emptypb.Empty{}, nil
+}

internal/storage/mysqlstore/mysql.go

@@ -46,10 +46,11 @@ type store struct {
 	archivist.UnimplementedArchivistServer
 	archivist.UnimplementedCollectorServer
 
-	client *ent.Client
+	client        *ent.Client
+	objectStorage archivist.CollectorServer
 }
 
-func NewServer(ctx context.Context, connectionstring string) (UnifiedStorage, chan error, error) {
+func NewServer(ctx context.Context, connectionstring string, objectStorage archivist.CollectorServer) (UnifiedStorage, <-chan error, error) {
 	drv, err := sql.Open("mysql", "root:example@tcp(db)/testify")
 	if err != nil {
 		return nil, nil, err
@@ -86,7 +87,8 @@ func NewServer(ctx context.Context, connectionstring string) (UnifiedStorage, ch
 	}
 
 	return &store{
-		client: client,
+		client:        client,
+		objectStorage: objectStorage,
 	}, errCh, nil
 }
 
@@ -188,7 +190,14 @@ func (s *store) Store(ctx context.Context, request *archivist.StoreRequest) (*em
 		return nil, err
 	}
 
-	fmt.Println("stored!")
+	fmt.Println("metadata stored")
+
+	_, err = s.objectStorage.Store(ctx, request)
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("object stored")
 
 	return &emptypb.Empty{}, nil
 }