Merge pull request #22 from deepfence/add-grpc

Add gRPC support

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "agent-plugins-grpc"]
+	path = agent-plugins-grpc
+	url = http://github.com/deepfence/agent-plugins-grpc

Dockerfile

@@ -1,22 +1,32 @@
 FROM golang:1.17-alpine3.13 AS builder
 MAINTAINER DeepFence
 
-RUN apk update && apk add --upgrade hyperscan-dev gcc musl-dev pkgconfig g++
+RUN apk update && apk add --upgrade hyperscan-dev gcc musl-dev pkgconfig g++ make git protoc
 ENV GOPATH=/root/.go \
     PKG_CONFIG_PATH=/usr/local/include/hs/ \
     CGO_CFLAGS="-I/usr/local/include/hyperscan/src" \
     LD_LIBRARY_PATH=/usr/local/lib:/usr/local/include/hs/lib:$LD_LIBRARY_PATH \
-    PATH=/usr/local/go-1.17.5/bin:~/.go/bin:$PATH
+    PATH=/usr/local/go-1.17.5/bin:/root/.go/bin:$PATH
+
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
+RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
+
+WORKDIR /home/deepfence/src/
+RUN git clone https://github.com/containerd/nerdctl
+WORKDIR /home/deepfence/src/nerdctl
+RUN make
 
 WORKDIR /home/deepfence/src/SecretScanner
 COPY . .
-RUN go build -v -i
+RUN make clean
+RUN make
 
 FROM alpine:3.13
 MAINTAINER DeepFence
 
 RUN apk update && apk add --upgrade libstdc++ libgcc docker hyperscan
 WORKDIR /home/deepfence/usr
+COPY --from=builder /home/deepfence/src/nerdctl/_output/nerdctl /bin
 COPY --from=builder /home/deepfence/src/SecretScanner/SecretScanner .
 COPY --from=builder /home/deepfence/src/SecretScanner/config.yaml .
 WORKDIR /home/deepfence/output

Makefile

@@ -0,0 +1,17 @@
+all: SecretScanner
+
+$(PWD)/agent-plugins-grpc/proto/*.proto:
+	$(PWD)/bootstrap.sh
+
+$(PWD)/agent-plugins-grpc/proto/*.go: $(PWD)/agent-plugins-grpc/proto/*.proto
+	(cd agent-plugins-grpc && make go)
+
+clean:
+	-(cd agent-plugins-grpc && make clean)
+	-rm ./SecretScanner
+
+SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/proto/*.go
+	go mod vendor
+	go build -v
+
+.PHONY: clean

README.md

@@ -46,6 +46,8 @@ Usage of ./SecretScanner:
     	Directory to process and store repositories/matches (default "/tmp")
   -threads int
     	Number of concurrent threads (default number of logical CPUs)
+  -socket-path string
+  		The gRPC server socket path
 
 ```
 
@@ -65,34 +67,58 @@ Install docker and run SecretScanner on a container image using the following in
 
 `docker pull node:8.11`
 
-* Run SecretScanner:
+* Run SecretScanner as a standalone:
   * Scan a container image:
 
     ```
-    docker run -it --rm --name=deepfence-secretscanner -v $(pwd):/home/deepfence/output -v /var/run/docker.sock:/var/run/docker.sock deepfenceio/secretscanning -image-name node:8.11
+    docker run -it --rm --name=deepfence-secretscanner -v $(pwd):/home/deepfence/output -v /var/run/docker.sock:/var/run/docker.sock -v /run/containerd/containerd.sock:/run/containerd/containerd.sock deepfenceio/secretscanning -image-name node:8.11
     ```
 
   * Scan a local directory:
 
     ```
-    docker run -it --rm --name=deepfence-secretscanner -v $(pwd):/home/deepfence/output -v /var/run/docker.sock:/var/run/docker.sock deepfenceio/secretscanning -local /home/deepfence/src/SecretScanner/test
+    docker run -it --rm --name=deepfence-secretscanner -v $(pwd):/home/deepfence/output -v /var/run/docker.sock:/var/run/docker.sock -v /run/containerd/containerd.sock:/run/containerd/containerd.sock deepfenceio/secretscanning -local /home/deepfence/src/SecretScanner/test
+    ```
+
+* Or run SecretScanner as a gRPC server:
+	```
+	docker run -it --rm --name=deepfence-secretscanner -v $(pwd):/home/deepfence/output -v /var/run/docker.sock:/var/run/docker.sock -v /run/containerd/containerd.sock:/run/containerd/containerd.sock -v /tmp/sock:/tmp/sock deepfenceio -socket-path /tmp/sock/s.sock
+
+	```
+  * Scan a container image:
+
+    ```
+	grpcurl -plaintext -import-path ./agent-plugins-grpc/proto -proto secret_scanner.proto -d '{"image": {"name": "node:8.11"}}' -unix '/tmp/sock.sock' secret_scanner.SecretScanner/FindSecretInfo
+
+    ```
+
+  * Scan a local directory:
+
+    ```
+	grpcurl -plaintext -import-path ./agent-plugins-grpc/proto -proto secret_scanner.proto -d '{"path": "/tmp"}' -unix '/tmp/sock.sock' secret_scanner.SecretScanner/FindSecretInfo
+
     ```
 
 By default, SecretScanner will also create json files with details of all the secrets found in the current working directory. You can explicitly specify the output directory and json filename using the appropriate options.
 
+Please note that you can use `nerdctl` as an alternative to `docker` in the commands above.
+
 # Build Instructions
 
-1. Install Docker
-2. Install Hyperscan
-3. Install go for your platform (version 1.14)
-4. Install go modules, if needed: `gohs`, `yaml.v3` and `color`
-5. `go get github.com/deepfence/SecretScanner` will download and build SecretScanner automatically in `$GOPATH/bin` or `$HOME/go/bin` directory. Or, clone this repository and run `go build -v -i` to build the executable in the current directory.
-6. Edit config.yaml file as needed and run the secret scanner with the appropriate config file directory.
+1. Run boostrap.sh
+2. Install Docker
+3. Install Hyperscan
+4. Install go for your platform (version 1.14)
+5. Install go modules, if needed: `gohs`, `yaml.v3` and `color`
+6. `go get github.com/deepfence/SecretScanner` will download and build SecretScanner automatically in `$GOPATH/bin` or `$HOME/go/bin` directory. Or, clone this repository and run `go build -v -i` to build the executable in the current directory.
+7. Edit config.yaml file as needed and run the secret scanner with the appropriate config file directory.
 
 For reference, the [Install file](https://github.com/deepfence/SecretScanner/blob/master/Install.Ubuntu) has commands to build on an ubuntu system.
 
 # Instructions to Run on Local Host
 
+## As a standalone application
+
 ```
 ./SecretScanner --help
 
@@ -101,6 +127,13 @@ For reference, the [Install file](https://github.com/deepfence/SecretScanner/blo
 ./SecretScanner -config-path /path/to/config.yaml/dir -image-name node:8.11
 ```
 
+## As a server application
+```
+./SecretScanner -socket-path /path/to/socket.sock
+```
+
+See "Quickly-Try-Using-Docker" section above to see how to send requests.
+
 # Sample SecretScanner Output
 
 ![SampleJsonOutput](images/SampleSecretsOutput.png)

bootstrap.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+git submodule update --init --remote --recursive ./agent-plugins-grpc

go.mod

@@ -3,13 +3,22 @@ module github.com/deepfence/SecretScanner
 go 1.17
 
 require (
+	github.com/deepfence/agent-plugins-grpc v0.0.0
 	github.com/fatih/color v1.13.0
 	github.com/flier/gohs v1.1.1
+	google.golang.org/grpc v1.41.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
 
 require (
+	github.com/deepfence/vessel v0.5.3
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/mattn/go-colorable v0.1.9 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
+	golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 // indirect
 	golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab // indirect
+	golang.org/x/text v0.3.6 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
 )
+
+replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc