Elasticsearch githubintegration index mappings are created at runtime

techjacker · Jul 14, 2017 · 2eaf3b7 · 2eaf3b7
1 parent 17da7b9
commit 2eaf3b7
Show file tree

Hide file tree

Showing 7 changed files with 151 additions and 32 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -12,3 +12,12 @@
     sha: cab517ac1132ea76603bd51ba5a95305f81bb2ba
     hooks:
     -   id: gofmt
+
+-   repo: local
+    hooks:
+    -   id: kubeval
+        name: kubeval
+        description: Lint kube files with system.
+        entry: kubeval
+        language: system
+        files: kube\/(?!secrets\.yml)
diff --git a/README.md b/README.md
@@ -49,41 +49,38 @@ these/pems/are/ok/*.pem
 ## Notifications
 Work in progress.
 
-
 ### Local Testing
-
 #### Set environment variables needed
 Create `env` file and update environment variables.
 ```
-cp .env{.example,}
-vi .env
+$ cp .env{.example,}
+# update .env values
+$ vi .env
+$ source .env
 ```
 
 #### Launch containers
 ```
-# launch server first
-. .env && docker-compose up -d server
-
-# Elastalert expects the elasticsearch index it is monitoring to exist otherwise it will error. The server creates the index in elasticsearch the first time it writes a log.
-make test-run-offenses
-
-# launch elastalert
-docker-compose up elastalert
+$ docker-compose up -d
 ```
 
-#### Debugging
+#### Run test offenses
+```
+$ make test-run-offenses
 ```
-docker exec -it <elastalert_container_hash> sh
 
-# run elastalert test rule utility within elastalert container
-elastalert-test-rule --config $ELASTALERT_CONFIG --count-only "$RULES_DIRECTORY/new_violation.yaml"
-elastalert-test-rule --alert --config $ELASTALERT_CONFIG "$RULES_DIRECTORY/new_violation.yaml"
 
+### Debugging Elastalert
+```
+$ docker exec -it <elastalert_container_hash> sh
+# run elastalert test rule utility within elastalert container
+$ elastalert-test-rule --config $ELASTALERT_CONFIG --count-only "$RULES_DIRECTORY/new_violation.yaml"
+$ elastalert-test-rule --alert --config $ELASTALERT_CONFIG "$RULES_DIRECTORY/new_violation.yaml"
 # run elastalert in debug mode
-elastalert --config "$ELASTALERT_CONFIG" --rule "$RULES_DIRECTORY/new_violation.yaml" --debug
+$ elastalert --config "$ELASTALERT_CONFIG" --rule "$RULES_DIRECTORY/new_violation.yaml" --debug
 ```
 
 #### Logs
 ```
-tail -f /log/elastalert_new_violation_rule.log
+$ tail -f /log/elastalert_new_violation_rule.log
 ```
diff --git a/elastalert/Dockerfile b/elastalert/Dockerfile
@@ -10,7 +10,7 @@ ENV CONTAINER_TIMEZONE Europe/London
 WORKDIR /opt
 RUN apk update && \
     apk upgrade && \
-    apk add ca-certificates openssl-dev libffi-dev python-dev gcc musl-dev tzdata openntpd && \
+    apk add ca-certificates openssl-dev libffi-dev python-dev gcc musl-dev tzdata openntpd curl && \
     rm -rf /var/cache/apk/* && \
 # Install pip - required for installation of Elastalert.
     wget https://bootstrap.pypa.io/get-pip.py && \
@@ -38,9 +38,7 @@ ENV ELASTALERT_CONFIG /opt/elastalert_config.yaml
 COPY elastalert_config.yaml ${ELASTALERT_CONFIG}
 COPY modules ${ELASTALERT_ROOT}/elastalert_modules
 
-
-
-
+COPY elasticsearch_mappings.json /tmp
 COPY start-elastalert.sh /opt/
 RUN chmod +x /opt/start-elastalert.sh
 CMD ["/opt/start-elastalert.sh"]
diff --git a/elastalert/elasticsearch_mappings.json b/elastalert/elasticsearch_mappings.json
@@ -0,0 +1,104 @@
+{
+    "mappings": {
+        "log": {
+            "properties": {
+                "Data": {
+                    "properties": {
+                        "filename": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "index": false,
+                                    "ignore_above": 256
+                                }
+                            },
+                            "fielddata": false
+                        },
+                        "organisation": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "index": false,
+                                    "ignore_above": 256
+                                }
+                            },
+                            "fielddata": false
+                        },
+                        "reason": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "index": false,
+                                    "ignore_above": 256
+                                }
+                            },
+                            "fielddata": false
+                        },
+                        "repo": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "index": false,
+                                    "ignore_above": 256
+                                }
+                            },
+                            "fielddata": false
+                        },
+                        "url": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "index": false,
+                                    "ignore_above": 256
+                                }
+                            },
+                            "fielddata": false
+                        }
+                    }
+                },
+                "Host": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "index": false,
+                            "ignore_above": 256
+                        }
+                    },
+                    "fielddata": false
+                },
+                "Level": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "index": false,
+                            "ignore_above": 256
+                        }
+                    },
+                    "fielddata": false
+                },
+                "Message": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "index": false,
+                            "ignore_above": 256
+                        }
+                    },
+                    "fielddata": false
+                },
+                "Timestamp": {
+                    "type": "date",
+                    "format": "strict_date_optional_time||epoch_millis"
+                }
+            }
+        }
+    }
+}
diff --git a/elastalert/start-elastalert.sh b/elastalert/start-elastalert.sh
@@ -1,6 +1,17 @@
 #!/bin/sh
 set -e
 
+####################
+# create elasticsearch mappings for index monitoring
+# elastlaert will throw an error if these do not already exist
+####################
+curl -s "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/githubintegration/_search"
+if [[ $? != 0 ]]; then
+    curl \
+        --upload-file /tmp/elasticsearch_mappings.json \
+        "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/githubintegration" | python -m json.tool
+fi
+
 ####################
 # Set the timezone
 ####################

diff --git a/handlers.go b/handlers.go
@@ -8,11 +8,11 @@ import (
 )
 
 const (
-	msgHealthOk   = "ok"
-	msgBadRequest = "bad request"
-	msgIgnore     = "Not a push evt; ignoring"
-	msgSuccess    = "Push contains no offenses"
-	msgFail       = "TODO: email list of recipients to be notified of violations"
+	msgHealthOk          = "ok"
+	msgBadRequest        = "bad request"
+	msgIgnore            = "Not a push evt; ignoring"
+	msgNoViolationsFound = "Push contains no offenses"
+	msgViolationFound    = "Push contains violations"
 )
 
 // GithubHandler is a github integration HTTP handler
@@ -43,7 +43,7 @@ func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP, log Log) http.Handler
 		}
 
 		if results.Matches() < 1 {
-			fmt.Fprintf(w, "%s\n", msgSuccess)
+			fmt.Fprintf(w, "%s\n", msgNoViolationsFound)
 			return
 		}
 
@@ -57,7 +57,7 @@ func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP, log Log) http.Handler
 				)
 			}
 		}
-		fmt.Fprintf(w, "%s\n", msgFail)
+		fmt.Fprintf(w, "%s\n", msgViolationFound)
 	})
 }
 

diff --git a/handlers_test.go b/handlers_test.go
@@ -64,7 +64,7 @@ func TestGithubHandler(t *testing.T) {
 				diffPath:          "test/fixtures/no_offenses.diff",
 			},
 			wantStatusCode: http.StatusOK,
-			wantResBody:    msgSuccess,
+			wantResBody:    msgNoViolationsFound,
 		},
 		{
 			name: "1 offense in diff",
@@ -74,7 +74,7 @@ func TestGithubHandler(t *testing.T) {
 				diffPath:          "test/fixtures/offenses_x1.diff",
 			},
 			wantStatusCode: http.StatusOK,
-			wantResBody:    msgFail,
+			wantResBody:    msgViolationFound,
 		},
 	}
 	for _, tt := range tests {