From 5d9878622164254c1c1b951d4a73cf150239635f Mon Sep 17 00:00:00 2001 From: Andrew Griffiths Date: Tue, 21 Mar 2017 12:58:03 +0000 Subject: [PATCH] feature/210-notifications-app: Adds elasticsearch logger --- .env | 1 + Dockerfile | 4 +- Makefile | 13 + cmd/scanrepo/main.go | 28 +-- docker-compose.yml | 30 +++ github.go | 4 + handlers.go | 22 +- handlers_test.go | 10 + hello | 0 log.go | 50 ++++ main.go | 29 ++- test/fixtures/github_event_push_offenses.json | 225 ++++++++++++++++++ vendor.conf | 2 +- version | 2 +- 14 files changed, 389 insertions(+), 31 deletions(-) create mode 100644 docker-compose.yml create mode 100644 hello create mode 100644 log.go create mode 100644 test/fixtures/github_event_push_offenses.json diff --git a/.env b/.env index 679fc1b..3e84590 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ export GITHUB_WEBHOOKSECRET=blah +export ELASTICSEARCH_URL="http://localhost:9200" diff --git a/Dockerfile b/Dockerfile index 132f9f9..0ef475c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,3 @@ -FROM golang:1.6-onbuild +FROM golang:1.8-onbuild - EXPOSE 8080 - diff --git a/Makefile b/Makefile index 4ea49ee..b502694 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ OFFENSES_X1 = f591c33a1b9500d0721b6664cfb6033d47a00793 FIXT_DIR = test/fixtures RULES_FILE = $(FIXT_DIR)/rules/gitrob.json DIFF_FILE = $(FIXT_DIR)/github_event_push.json +DIFF_FILE_OFFENSES = $(FIXT_DIR)/github_event_push_offenses.json RULES_URL = https://raw.githubusercontent.com/michenriksen/gitrob/master/signatures.json cli: @@ -56,6 +57,10 @@ run: mac-diff-file: @cat $(DIFF_FILE) | openssl sha1 -hmac $(GITHUB_SECRET) | sed 's/^.* //' +mac-diff-file-offenses: + @cat $(DIFF_FILE_OFFENSES) | openssl sha1 -hmac $(GITHUB_SECRET) | sed 's/^.* //' + + test-run: @wget -O- \ -X POST \ @@ -64,6 +69,14 @@ test-run: --post-file "$(DIFF_FILE)" \ http://localhost:$(PORT)/github +test-run-offenses: + @wget -O- \ + -X POST \ + --header="X-GitHub-Event: push" \ + --header="X-Hub-Signature: sha1=$(shell make mac-diff-file-offenses)" \ + --post-file "$(DIFF_FILE_OFFENSES)" \ + http://localhost:$(PORT)/github + test-run-fail: @wget -O- \ -X POST \ diff --git a/cmd/scanrepo/main.go b/cmd/scanrepo/main.go index 364622d..34ed032 100644 --- a/cmd/scanrepo/main.go +++ b/cmd/scanrepo/main.go @@ -47,20 +47,20 @@ func main() { } matches := res.Matches() - if matches > 0 { - i := 1 - fmt.Printf("Diff contains %d offenses\n\n", matches) - for filename, rule := range res.MatchedRules { - fmt.Printf("------------------\n") - fmt.Printf("Violation %d\n", i) - fmt.Printf("File: %s\n", filename) - fmt.Printf("Reason: %#v\n\n", rule[0].Caption) - i++ - } - // finding violations constitutes an error - os.Exit(1) + if matches < 1 { + fmt.Printf("Diff contains NO offenses\n\n") return } - fmt.Printf("Diff contains NO offenses\n\n") - os.Exit(0) + + i := 1 + fmt.Fprintf(os.Stderr, "Diff contains %d offenses\n\n", matches) + for filename, rule := range res.MatchedRules { + fmt.Fprintf(os.Stderr, "------------------\n") + fmt.Fprintf(os.Stderr, "Violation %d\n", i) + fmt.Fprintf(os.Stderr, "File: %s\n", filename) + fmt.Fprintf(os.Stderr, "Reason: %#v\n\n", rule[0].Caption) + i++ + } + // finding violations constitutes an error + os.Exit(1) } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..49fde13 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +--- +version: "2.1" + +services: + + server: + build: + context: . + env_file: .env + environment: + - GITHUB_WEBHOOKSECRET + - ELASTICSEARCH_URL=http://elasticsearch:9200 + depends_on: + elasticsearch: + condition: service_healthy + ports: + - 8080:8080 + + elasticsearch: + restart: always + image: elasticsearch:2 + ports: + - 9200:9200 + expose: + - "9200" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:9200"] + interval: 5s + timeout: 3s + retries: 5 diff --git a/github.go b/github.go index c4fb1eb..a8d3a01 100644 --- a/github.go +++ b/github.go @@ -31,6 +31,7 @@ func DecodeJSON(r io.Reader, v interface{}) error { // GithubResponse is for marshalling Github push event JSON payloads type GithubResponse struct { + Compare string Commits []struct { Added []string `json:"added"` ID string `json:"id"` @@ -46,6 +47,9 @@ type GithubResponse struct { // OK validates a marshalled struct func (g *GithubResponse) OK() error { + if len(g.Compare) < 1 { + return errors.New("missing compare URL") + } if len(g.Commits) < 1 { return errors.New("empty payload") } diff --git a/handlers.go b/handlers.go index c14155a..e0bda2d 100644 --- a/handlers.go +++ b/handlers.go @@ -16,7 +16,7 @@ const ( ) // GithubHandler is a github integration HTTP handler -func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP) http.Handler { +func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP, log Log) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // decode github push event payload gitRes := &GithubResponse{} @@ -41,13 +41,23 @@ func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP) http.Handler { } results = append(results, diffRes) } - // TODO: Notify recipients if fails checks - // stringify results vs pass to logger - if results.Matches() > 0 { - fmt.Fprintf(w, "%s\n", msgFail) + + if results.Matches() < 1 { + fmt.Fprintf(w, "%s\n", msgSuccess) return } - fmt.Fprintf(w, "%s\n", msgSuccess) + + for _, res := range results { + if res.Matches() > 0 { + log.Log( + res.MatchedRules, + gitRes.Repository.Owner.Name, + gitRes.Repository.Name, + gitRes.Compare, + ) + } + } + fmt.Fprintf(w, "%s\n", msgFail) }) } diff --git a/handlers_test.go b/handlers_test.go index 3459b4b..3d439b2 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/Sirupsen/logrus" "github.com/julienschmidt/httprouter" "github.com/techjacker/diffence" ) @@ -21,6 +22,14 @@ func (t testDiffGetter) Get(_ string) (*http.Response, error) { }, nil } +type testLogger struct { + log *logrus.Logger +} + +func (l testLogger) Log(v ...interface{}) { + return +} + func TestGithubHandler(t *testing.T) { const ( testPath = "/github" @@ -75,6 +84,7 @@ func TestGithubHandler(t *testing.T) { router.Handler("POST", testPath, GithubHandler( diffence.DiffChecker{Rules: getTestRules(t, tt.args.rulesPath)}, testDiffGetter{tt.args.diffPath}, + testLogger{}, )) params := getFixture(tt.args.githubPayloadPath) diff --git a/hello b/hello new file mode 100644 index 0000000..e69de29 diff --git a/log.go b/log.go new file mode 100644 index 0000000..c0b7456 --- /dev/null +++ b/log.go @@ -0,0 +1,50 @@ +package main + +import ( + "fmt" + + "gopkg.in/olivere/elastic.v3" + "gopkg.in/sohlich/elogrus.v1" + + "github.com/Sirupsen/logrus" + "github.com/techjacker/diffence" +) + +// Log is the log interface for the app +type Log interface { + Log(v ...interface{}) +} + +// Log is the log for the app +type Logger struct { + log *logrus.Logger +} + +func (l Logger) Log(v ...interface{}) { + i := 1 + for filename, rule := range v[0].(diffence.MatchedRules) { + i++ + l.log.WithFields(logrus.Fields{ + "organisation": v[1], + "repo": v[2], + "url": v[3], + "filename": filename, + "reason": rule[0].Caption, + }).Error(fmt.Sprintf("Violation found in %s:%s", v[1], v[2])) + } +} + +// NewESLogger is a factory for Elasticsearch loggers +func NewESLogger(esUrl, esIndex string) (Logger, error) { + log := logrus.New() + client, err := elastic.NewClient(elastic.SetURL(esUrl)) + if err != nil { + return Logger{}, err + } + hook, err := elogrus.NewElasticHook(client, "localhost", logrus.DebugLevel, esIndex) + if err != nil { + return Logger{}, err + } + log.Hooks.Add(hook) + return Logger{log: log}, nil +} diff --git a/main.go b/main.go index eeba492..ae2333d 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "path" "runtime" + "github.com/Sirupsen/logrus" "github.com/julienschmidt/httprouter" "github.com/techjacker/diffence" ) @@ -28,12 +29,27 @@ func getRules(rulesPath string) *[]diffence.Rule { return rules } -func getEnvVar(name string) []byte { - secretStr := os.Getenv(name) - if len(secretStr) < 1 { +func getRequiredEnvVar(name string) []byte { + val, ok := os.LookupEnv(name) + if !ok { panic(fmt.Sprintf("Env var:%s not set in environment", name)) } - return []byte(secretStr) + return []byte(val) +} + +func getLogger() Log { + esURL, ok := os.LookupEnv("ELASTICSEARCH_URL") + if ok { + fmt.Printf("Logging to elasticsearch: %s\n", esURL) + logger, err := NewESLogger(esURL, "githubintegration") + if err != nil { + panic(err) + } + return logger + } + return Logger{ + log: logrus.New(), + } } func main() { @@ -43,10 +59,11 @@ func main() { GithubHandler( diffence.DiffChecker{Rules: getRules(gitrobRules)}, diffGetterGithub{}, + getLogger(), ), - AuthMiddleware(GithubAuthenticator{getEnvVar(githubWebhookSecret)}), + AuthMiddleware(GithubAuthenticator{getRequiredEnvVar(githubWebhookSecret)}), )) - fmt.Printf("Server listening on port: %d", serverPort) + fmt.Printf("Server listening on port: %d\n", serverPort) http.ListenAndServe(fmt.Sprintf(":%d", serverPort), router) } diff --git a/test/fixtures/github_event_push_offenses.json b/test/fixtures/github_event_push_offenses.json new file mode 100644 index 0000000..ff26028 --- /dev/null +++ b/test/fixtures/github_event_push_offenses.json @@ -0,0 +1,225 @@ +{ + "ref": "refs/heads/master", + "before": "3cbc791b208b9557fbf61d83a2af057537d909b5", + "after": "ae842c75ecb4ffacc888d7a83304c157a229625c", + "created": false, + "deleted": false, + "forced": true, + "base_ref": null, + "compare": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/compare/3cbc791b208b...ae842c75ecb4", + "commits": [ + { + "id": "b421aaf0767e236aaeb20258f0a70d2228b83f16", + "tree_id": "9bc182221a87f1d849d3e84c011d3afc587a01b9", + "distinct": true, + "message": "18", + "timestamp": "2017-02-21T20:13:03Z", + "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/b421aaf0767e236aaeb20258f0a70d2228b83f16", + "author": { + "name": "Andrew Griffiths", + "email": "mail@andrewgriffithsonline.com", + "username": "techjacker" + }, + "committer": { + "name": "Andrew Griffiths", + "email": "mail@andrewgriffithsonline.com", + "username": "techjacker" + }, + "added": [ + "18" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + { + "id": "ae842c75ecb4ffacc888d7a83304c157a229625c", + "tree_id": "ddba7ffb347ca2934b2ed0891bb06b5b777b0f4d", + "distinct": true, + "message": "secrets99", + "timestamp": "2017-03-15T22:21:30Z", + "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/ae842c75ecb4ffacc888d7a83304c157a229625c", + "author": { + "name": "Andrew Griffiths", + "email": "mail@andrewgriffithsonline.com", + "username": "techjacker" + }, + "committer": { + "name": "Andrew Griffiths", + "email": "mail@andrewgriffithsonline.com", + "username": "techjacker" + }, + "added": [ + "secrets99" + ], + "removed": [ + + ], + "modified": [ + + ] + } + ], + "head_commit": { + "id": "ae842c75ecb4ffacc888d7a83304c157a229625c", + "tree_id": "ddba7ffb347ca2934b2ed0891bb06b5b777b0f4d", + "distinct": true, + "message": "secrets99", + "timestamp": "2017-03-15T22:21:30Z", + "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/ae842c75ecb4ffacc888d7a83304c157a229625c", + "author": { + "name": "Andrew Griffiths", + "email": "mail@andrewgriffithsonline.com", + "username": "techjacker" + }, + "committer": { + "name": "Andrew Griffiths", + "email": "mail@andrewgriffithsonline.com", + "username": "techjacker" + }, + "added": [ + "secrets99" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + "repository": { + "id": 79224801, + "name": "testgithubintegration", + "full_name": "ukhomeoffice-bot-test/testgithubintegration", + "owner": { + "name": "ukhomeoffice-bot-test", + "email": null, + "login": "ukhomeoffice-bot-test", + "id": 22323467, + "avatar_url": "https://avatars1.githubusercontent.com/u/22323467?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/ukhomeoffice-bot-test", + "html_url": "https://github.com/ukhomeoffice-bot-test", + "followers_url": "https://api.github.com/users/ukhomeoffice-bot-test/followers", + "following_url": "https://api.github.com/users/ukhomeoffice-bot-test/following{/other_user}", + "gists_url": "https://api.github.com/users/ukhomeoffice-bot-test/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ukhomeoffice-bot-test/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ukhomeoffice-bot-test/subscriptions", + "organizations_url": "https://api.github.com/users/ukhomeoffice-bot-test/orgs", + "repos_url": "https://api.github.com/users/ukhomeoffice-bot-test/repos", + "events_url": "https://api.github.com/users/ukhomeoffice-bot-test/events{/privacy}", + "received_events_url": "https://api.github.com/users/ukhomeoffice-bot-test/received_events", + "type": "Organization", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", + "description": null, + "fork": false, + "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", + "forks_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/forks", + "keys_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/teams", + "hooks_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/hooks", + "issue_events_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues/events{/number}", + "events_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/events", + "assignees_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/assignees{/user}", + "branches_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/branches{/branch}", + "tags_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/tags", + "blobs_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/statuses/{sha}", + "languages_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/languages", + "stargazers_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/stargazers", + "contributors_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/contributors", + "subscribers_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/subscribers", + "subscription_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/subscription", + "commits_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/contents/{+path}", + "compare_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/merges", + "archive_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/downloads", + "issues_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues{/number}", + "pulls_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/pulls{/number}", + "milestones_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/milestones{/number}", + "notifications_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/labels{/name}", + "releases_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/releases{/id}", + "deployments_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/deployments", + "created_at": 1484655622, + "updated_at": "2017-01-17T12:20:22Z", + "pushed_at": 1489616498, + "git_url": "git://github.com/ukhomeoffice-bot-test/testgithubintegration.git", + "ssh_url": "git@github.com:ukhomeoffice-bot-test/testgithubintegration.git", + "clone_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration.git", + "svn_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", + "homepage": null, + "size": 8, + "stargazers_count": 0, + "watchers_count": 0, + "language": null, + "has_issues": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "open_issues_count": 0, + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "ukhomeoffice-bot-test" + }, + "pusher": { + "name": "techjacker", + "email": "mail@andrewgriffithsonline.com" + }, + "organization": { + "login": "ukhomeoffice-bot-test", + "id": 22323467, + "url": "https://api.github.com/orgs/ukhomeoffice-bot-test", + "repos_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/repos", + "events_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/events", + "hooks_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/hooks", + "issues_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/issues", + "members_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/members{/member}", + "public_members_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/public_members{/member}", + "avatar_url": "https://avatars1.githubusercontent.com/u/22323467?v=3", + "description": null + }, + "sender": { + "login": "techjacker", + "id": 923307, + "avatar_url": "https://avatars2.githubusercontent.com/u/923307?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/techjacker", + "html_url": "https://github.com/techjacker", + "followers_url": "https://api.github.com/users/techjacker/followers", + "following_url": "https://api.github.com/users/techjacker/following{/other_user}", + "gists_url": "https://api.github.com/users/techjacker/gists{/gist_id}", + "starred_url": "https://api.github.com/users/techjacker/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/techjacker/subscriptions", + "organizations_url": "https://api.github.com/users/techjacker/orgs", + "repos_url": "https://api.github.com/users/techjacker/repos", + "events_url": "https://api.github.com/users/techjacker/events{/privacy}", + "received_events_url": "https://api.github.com/users/techjacker/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 5873 + } +} diff --git a/vendor.conf b/vendor.conf index ebe8249..f4abeaa 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,5 +1,5 @@ # package github.com/UKHomeOffice/repo-security-scanner -github.com/techjacker/diffence 856a7c6bbc254d8086278d75dd8981fcc72d979b +github.com/techjacker/diffence 12c4abbcb3c837367860b4bfa55f01e86dc5b377 github.com/julienschmidt/httprouter 8a45e95fc75cb77048068a62daed98cc22fdac7c diff --git a/version b/version index 73537c2..b241ff9 100644 --- a/version +++ b/version @@ -1,2 +1,2 @@ -RELEASE_VERSION=0.3.0 +RELEASE_VERSION=0.3.1 RELEASE_BUILD_PATH=./cmd/scanrepo