Implement Elasticsearch Index API

sematext · Oct 15, 2017 · 34d1f69 · 34d1f69
1 parent 7b40cd9
commit 34d1f69
Show file tree

Hide file tree

Showing 22 changed files with 204 additions and 261 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,7 +8,7 @@ pids
 *.pid
 *.seed
 .vscode
-
+package-lock.json
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 

diff --git a/bin/logagent.js b/bin/logagent.js
@@ -37,6 +37,7 @@ var moduleAlias = {
   'input-kafka': '../lib/plugins/input/kafka.js',
   'input-influxdb-http': '../lib/plugins/input/influxHttp.js',
   'elasticsearch-query': '../lib/plugins/input/elasticsearchQuery.js',
+  'input-elasticsearch-http': '../lib/plugins/input/elasticsearchHttp.js',
 
   // input filters
   grep: '../lib/plugins/input-filter/grep.js',

diff --git a/config/examples/elastic-http-input.yml b/config/examples/elastic-http-input.yml
@@ -0,0 +1,15 @@
+# Global options
+options:
+  includeOriginalLine: true
+
+input:
+   elasticsearch-http:
+    module: input-elasticsearch-http
+    port: 9900
+    worker: 2
+
+output: 
+  stdout: yaml
+  elasticsearch:
+   module: elasticsearch
+   url: http://localhost:9200
diff --git a/elasticsearch-http-test/docker-compose.yml b/elasticsearch-http-test/docker-compose.yml
@@ -0,0 +1,23 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
+    container_name: elasticsearch
+    environment:
+      - path.data=/tmp/elasticsearch
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 1g
+    volumes:
+      - /tmp/elasticsearch/data:/tmp/elasticsearch
+    ports:
+      - "9200:9200"
+      - "9300:9300"
diff --git a/elasticsearch-http-test/requests b/elasticsearch-http-test/requests
@@ -0,0 +1,2 @@
+{ "index" : { "_index" : "mytest", "_type" : "type1", "_id" : "1" } }
+{ "field1":"value1" }
diff --git a/elasticsearch-http-test/test.sh b/elasticsearch-http-test/test.sh
@@ -0,0 +1,13 @@
+cd elasticsearch-test
+docker-compose version 
+docker version
+if [[ "$(docker ps)" =~ "elastic" ]]; then 
+  docker-compose stop && true
+  docker-compose rm -f && true
+fi 
+docker-compose up -d 
+sleep 25
+
+#curl -s -H "Content-Type: application/x-ndjson" -XPOST localhost:9900/_bulk --data-binary "@requests
+#sleep 10
+curl 'localhost:9200/_cat/indices?v' 2>&1 | grep mytest
diff --git a/kafka-test/README.txt b/kafka-test/README.txt
diff --git a/kafka-test/certs/ca-cert b/kafka-test/certs/ca-cert
diff --git a/kafka-test/certs/ca-cert.srl b/kafka-test/certs/ca-cert.srl
diff --git a/kafka-test/certs/ca-key b/kafka-test/certs/ca-key
diff --git a/kafka-test/certs/cert-file b/kafka-test/certs/cert-file
diff --git a/kafka-test/certs/cert-signed b/kafka-test/certs/cert-signed
diff --git a/kafka-test/certs/client.truststore.jks b/kafka-test/certs/client.truststore.jks
diff --git a/kafka-test/certs/server.keystore.jks b/kafka-test/certs/server.keystore.jks
diff --git a/kafka-test/certs/server.truststore.jks b/kafka-test/certs/server.truststore.jks
diff --git a/kafka-test/consumer-ssl.js b/kafka-test/consumer-ssl.js
diff --git a/kafka-test/consumer.js b/kafka-test/consumer.js
diff --git a/kafka-test/docker-compose.yml b/kafka-test/docker-compose.yml
diff --git a/kafka-test/producer-ssl.js b/kafka-test/producer-ssl.js
diff --git a/kafka-test/producer.js b/kafka-test/producer.js
diff --git a/kafka-test/start-docker-test.sh b/kafka-test/start-docker-test.sh