FLiP-Elastic
Get your Apache Pulsar + Elastic
Command Line Setup
docker network ls
docker network inspect elastic
curl -s http://pulsar1:9200/my_index/_search -u elastic:changeme
curl -s http://pulsar1:9200/my_index/_refresh -u elastic:changeme
docker logs -f docker-elk_elasticsearch_1
docker logs -f docker-elk_kibana_1
docker-compose up -d
docker ps
docker-compose ps
Deploy Sink
bin/pulsar-admin sinks stop --tenant public --namespace default --name elasticsearch-sink
bin/pulsar-admin sinks delete --tenant public --namespace default --name elasticsearch-sink
bin/pulsar-admin sinks create --archive ./connectors/pulsar-io-elastic-search-2.9.1.nar --tenant public
--namespace default
--name elasticsearch-sink --sink-config-file conf/elastic-sink.conf
--inputs persistent://public/default/iotjetsonjson --parallelism 1
bin/pulsar-admin sinks status --tenant public --namespace default --name elasticsearch-sink
{
"numInstances" : 1,
"numRunning" : 1,
"instances" : [ {
"instanceId" : 0,
"status" : {
"running" : true,
"error" : "",
"numRestarts" : 0,
"numReadFromPulsar" : 6,
"numSystemExceptions" : 0,
"latestSystemExceptions" : [ ],
"numSinkExceptions" : 0,
"latestSinkExceptions" : [ ],
"numWrittenToSink" : 6,
"lastReceivedTime" : 1643308578837,
"workerId" : "c-standalone-fw-127.0.0.1-8080"
}
} ]
}
bin/pulsar-admin sinks get --tenant public --namespace default --name elasticsearch-sink
{
"tenant": "public",
"namespace": "default",
"name": "elasticsearch-sink",
"className": "org.apache.pulsar.io.elasticsearch.ElasticSearchSink",
"sourceSubscriptionPosition": "Latest",
"inputs": [
"persistent://public/default/iotjetsonjson"
],
"inputSpecs": {
"persistent://public/default/iotjetsonjson": {
"isRegexPattern": false,
"schemaProperties": {},
"consumerProperties": {},
"poolMessages": false
}
},
"configs": {
"password": "changeme",
"elasticSearchUrl": "http://pulsar1:9200",
"indexName": "my_index",
"username": "elastic"
},
"parallelism": 1,
"processingGuarantees": "ATLEAST_ONCE",
"retainOrdering": false,
"autoAck": true
}
tail -500 logs/functions/public/default/elasticsearch-sink/elasticsearch-sink-0.log
2022-01-27T12:49:43,662-0500 [pulsar-client-io-1-2] INFO org.apache.pulsar.client.impl.ConsumerImpl - [persistent://public/default/iotjetsonjson][public/default/elasticsearch-sink] Subscribed to topic on 127.0.0.1/127.0.0.1:6650 -- consumer: 0
2022-01-27T13:26:12,202-0500 [pulsar-client-io-1-2] INFO com.scurrilous.circe.checksum.Crc32cIntChecksum - SSE4.2 CRC32C provider initialized
2022-01-27T13:26:12,239-0500 [public/default/elasticsearch-sink-0] INFO org.apache.pulsar.client.impl.schema.AutoConsumeSchema - Configure topic schema \x00\x00\x00\x00\x00\x00\x00\x00 for topic persistent://public/default/iotjetsonjson : {"type":"record","name":"IoTMessage","namespace":"io.streamnative.examples.oauth2","fields":[{"name":"camera","type":["null","string"],"default":null},{"name":"cpu","type":"double"},{"name":"cputemp","type":["null","string"],"default":null},{"name":"cputempf","type":["null","string"],"default":null},{"name":"diskusage","type":["null","string"],"default":null},{"name":"filename","type":["null","string"],"default":null},{"name":"gputemp","type":["null","string"],"default":null},{"name":"gputempf","type":["null","string"],"default":null},{"name":"host","type":["null","string"],"default":null},{"name":"host_name","type":["null","string"],"default":null},{"name":"imageinput","type":["null","string"],"default":null},{"name":"ipaddress","type":["null","string"],"default":null},{"name":"macaddress","type":["null","string"],"default":null},{"name":"memory","type":"double"},{"name":"networktime","type":"double"},{"name":"runtime","type":["null","string"],"default":null},{"name":"systemtime","type":["null","string"],"default":null},{"name":"te","type":["null","string"],"default":null},{"name":"top1","type":["null","string"],"default":null},{"name":"top1pct","type":"double"},{"name":"uuid","type":["null","string"],"default":null}]}
http://pulsar1:9200/my_index
{"my_index":{"aliases":{},"mappings":{"properties":{"camera":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"cpu":{"type":"float"},"cputemp":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"cputempf":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"diskusage":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"filename":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"gputemp":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"gputempf":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"host":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"host_name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"imageinput":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"ipaddress":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"macaddress":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"memory":{"type":"float"},"networktime":{"type":"float"},"runtime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"systemtime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"te":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"top1":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"top1pct":{"type":"float"},"uuid":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"settings":{"index":{"routing":{"allocation":{"include":{"_tier_preference":"data_content"}}},"number_of_shards":"1","provided_name":"my_index","creation_date":"1643307972382","number_of_replicas":"1","uuid":"fkczbchfTECNFdH-DKXYNg","version":{"created":"7160399"}}}}}
To Shutdown
docker-compose down
References
- https://pulsar.apache.org/docs/en/io-elasticsearch/
- https://www.youtube.com/watch?v=7IstxhL8p9E
- https://www.elastic.co/guide/en/kibana/current/docker.html
- https://www.elastic.co/blog/docker-networking
- https://www.elastic.co/guide/en/elasticsearch/reference/7.16/security-minimal-setup.html
- https://github.com/elastic/stack-docs/blob/main/docs/en/getting-started/get-started-docker.asciidoc
- https://www.elastic.co/guide/en/elasticsearch/reference/7.16/docker.html
- https://pulsar.apache.org/docs/en/io-elasticsearch-sink/
- https://github.com/deviantony/docker-elk