Merge pull request #9 from UKHomeOffice/feature/210-notifications-app

Adds elasticsearch logger

.env

@@ -1 +1,2 @@
 export GITHUB_WEBHOOKSECRET=blah
+export ELASTICSEARCH_URL="http://localhost:9200"

Dockerfile

@@ -1,5 +1,3 @@
-FROM golang:1.6-onbuild
+FROM golang:1.8-onbuild
 
-
 EXPOSE 8080
-

Makefile

@@ -10,6 +10,7 @@ OFFENSES_X1 = f591c33a1b9500d0721b6664cfb6033d47a00793
 FIXT_DIR = test/fixtures
 RULES_FILE = $(FIXT_DIR)/rules/gitrob.json
 DIFF_FILE = $(FIXT_DIR)/github_event_push.json
+DIFF_FILE_OFFENSES = $(FIXT_DIR)/github_event_push_offenses.json
 RULES_URL = https://raw.githubusercontent.com/michenriksen/gitrob/master/signatures.json
 
 cli:
@@ -56,6 +57,10 @@ run:
 mac-diff-file:
 	@cat $(DIFF_FILE) | openssl sha1 -hmac $(GITHUB_SECRET) | sed 's/^.* //'
 
+mac-diff-file-offenses:
+	@cat $(DIFF_FILE_OFFENSES) | openssl sha1 -hmac $(GITHUB_SECRET) | sed 's/^.* //'
+
+
 test-run:
 	@wget -O- \
 		-X POST \
@@ -64,6 +69,14 @@ test-run:
 		--post-file "$(DIFF_FILE)" \
 		http://localhost:$(PORT)/github
 
+test-run-offenses:
+	@wget -O- \
+		-X POST \
+		--header="X-GitHub-Event: push" \
+		--header="X-Hub-Signature: sha1=$(shell make mac-diff-file-offenses)" \
+		--post-file "$(DIFF_FILE_OFFENSES)" \
+		http://localhost:$(PORT)/github
+
 test-run-fail:
 	@wget -O- \
 		-X POST \

cmd/scanrepo/main.go

@@ -47,20 +47,20 @@ func main() {
 	}
 
 	matches := res.Matches()
-	if matches > 0 {
-		i := 1
-		fmt.Printf("Diff contains %d offenses\n\n", matches)
-		for filename, rule := range res.MatchedRules {
-			fmt.Printf("------------------\n")
-			fmt.Printf("Violation %d\n", i)
-			fmt.Printf("File: %s\n", filename)
-			fmt.Printf("Reason: %#v\n\n", rule[0].Caption)
-			i++
-		}
-		// finding violations constitutes an error
-		os.Exit(1)
+	if matches < 1 {
+		fmt.Printf("Diff contains NO offenses\n\n")
 		return
 	}
-	fmt.Printf("Diff contains NO offenses\n\n")
-	os.Exit(0)
+
+	i := 1
+	fmt.Fprintf(os.Stderr, "Diff contains %d offenses\n\n", matches)
+	for filename, rule := range res.MatchedRules {
+		fmt.Fprintf(os.Stderr, "------------------\n")
+		fmt.Fprintf(os.Stderr, "Violation %d\n", i)
+		fmt.Fprintf(os.Stderr, "File: %s\n", filename)
+		fmt.Fprintf(os.Stderr, "Reason: %#v\n\n", rule[0].Caption)
+		i++
+	}
+	// finding violations constitutes an error
+	os.Exit(1)
 }

docker-compose.yml

@@ -0,0 +1,30 @@
+---
+version: "2.1"
+
+services:
+
+  server:
+    build:
+      context: .
+    env_file: .env
+    environment:
+      - GITHUB_WEBHOOKSECRET
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    ports:
+      - 8080:8080
+
+  elasticsearch:
+    restart: always
+    image: elasticsearch:2
+    ports:
+      - 9200:9200
+    expose:
+      - "9200"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:9200"]
+      interval: 5s
+      timeout: 3s
+      retries: 5

github.go

@@ -31,6 +31,7 @@ func DecodeJSON(r io.Reader, v interface{}) error {
 
 // GithubResponse is for marshalling Github push event JSON payloads
 type GithubResponse struct {
+	Compare string
 	Commits []struct {
 		Added []string `json:"added"`
 		ID    string   `json:"id"`
@@ -46,6 +47,9 @@ type GithubResponse struct {
 
 // OK validates a marshalled struct
 func (g *GithubResponse) OK() error {
+	if len(g.Compare) < 1 {
+		return errors.New("missing compare URL")
+	}
 	if len(g.Commits) < 1 {
 		return errors.New("empty payload")
 	}

handlers.go

@@ -16,7 +16,7 @@ const (
 )
 
 // GithubHandler is a github integration HTTP handler
-func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP) http.Handler {
+func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP, log Log) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// decode github push event payload
 		gitRes := &GithubResponse{}
@@ -41,13 +41,23 @@ func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP) http.Handler {
 			}
 			results = append(results, diffRes)
 		}
-		// TODO: Notify recipients if fails checks
-		// stringify results vs pass to logger
-		if results.Matches() > 0 {
-			fmt.Fprintf(w, "%s\n", msgFail)
+
+		if results.Matches() < 1 {
+			fmt.Fprintf(w, "%s\n", msgSuccess)
 			return
 		}
-		fmt.Fprintf(w, "%s\n", msgSuccess)
+
+		for _, res := range results {
+			if res.Matches() > 0 {
+				log.Log(
+					res.MatchedRules,
+					gitRes.Repository.Owner.Name,
+					gitRes.Repository.Name,
+					gitRes.Compare,
+				)
+			}
+		}
+		fmt.Fprintf(w, "%s\n", msgFail)
 	})
 }
 

handlers_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/julienschmidt/httprouter"
 	"github.com/techjacker/diffence"
 )
@@ -21,6 +22,14 @@ func (t testDiffGetter) Get(_ string) (*http.Response, error) {
 	}, nil
 }
 
+type testLogger struct {
+	log *logrus.Logger
+}
+
+func (l testLogger) Log(v ...interface{}) {
+	return
+}
+
 func TestGithubHandler(t *testing.T) {
 	const (
 		testPath = "/github"
@@ -75,6 +84,7 @@ func TestGithubHandler(t *testing.T) {
 			router.Handler("POST", testPath, GithubHandler(
 				diffence.DiffChecker{Rules: getTestRules(t, tt.args.rulesPath)},
 				testDiffGetter{tt.args.diffPath},
+				testLogger{},
 			))
 
 			params := getFixture(tt.args.githubPayloadPath)

log.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+
+	"gopkg.in/olivere/elastic.v3"
+	"gopkg.in/sohlich/elogrus.v1"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/techjacker/diffence"
+)
+
+// Log is the log interface for the app
+type Log interface {
+	Log(v ...interface{})
+}
+
+// Log is the log for the app
+type Logger struct {
+	log *logrus.Logger
+}
+
+func (l Logger) Log(v ...interface{}) {
+	i := 1
+	for filename, rule := range v[0].(diffence.MatchedRules) {
+		i++
+		l.log.WithFields(logrus.Fields{
+			"organisation": v[1],
+			"repo":         v[2],
+			"url":          v[3],
+			"filename":     filename,
+			"reason":       rule[0].Caption,
+		}).Error(fmt.Sprintf("Violation found in %s:%s", v[1], v[2]))
+	}
+}
+
+// NewESLogger is a factory for Elasticsearch loggers
+func NewESLogger(esUrl, esIndex string) (Logger, error) {
+	log := logrus.New()
+	client, err := elastic.NewClient(elastic.SetURL(esUrl))
+	if err != nil {
+		return Logger{}, err
+	}
+	hook, err := elogrus.NewElasticHook(client, "localhost", logrus.DebugLevel, esIndex)
+	if err != nil {
+		return Logger{}, err
+	}
+	log.Hooks.Add(hook)
+	return Logger{log: log}, nil
+}

main.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"runtime"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/julienschmidt/httprouter"
 	"github.com/techjacker/diffence"
 )
@@ -28,12 +29,27 @@ func getRules(rulesPath string) *[]diffence.Rule {
 	return rules
 }
 
-func getEnvVar(name string) []byte {
-	secretStr := os.Getenv(name)
-	if len(secretStr) < 1 {
+func getRequiredEnvVar(name string) []byte {
+	val, ok := os.LookupEnv(name)
+	if !ok {
 		panic(fmt.Sprintf("Env var:%s not set in environment", name))
 	}
-	return []byte(secretStr)
+	return []byte(val)
+}
+
+func getLogger() Log {
+	esURL, ok := os.LookupEnv("ELASTICSEARCH_URL")
+	if ok {
+		fmt.Printf("Logging to elasticsearch: %s\n", esURL)
+		logger, err := NewESLogger(esURL, "githubintegration")
+		if err != nil {
+			panic(err)
+		}
+		return logger
+	}
+	return Logger{
+		log: logrus.New(),
+	}
 }
 
 func main() {
@@ -43,10 +59,11 @@ func main() {
 		GithubHandler(
 			diffence.DiffChecker{Rules: getRules(gitrobRules)},
 			diffGetterGithub{},
+			getLogger(),
 		),
-		AuthMiddleware(GithubAuthenticator{getEnvVar(githubWebhookSecret)}),
+		AuthMiddleware(GithubAuthenticator{getRequiredEnvVar(githubWebhookSecret)}),
 	))
 
-	fmt.Printf("Server listening on port: %d", serverPort)
+	fmt.Printf("Server listening on port: %d\n", serverPort)
 	http.ListenAndServe(fmt.Sprintf(":%d", serverPort), router)
 }