Kafka ingestion

Dockerfile

@@ -8,10 +8,17 @@ RUN apt-get update
 # Auto-accept the Oracle JDK license
 RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
 
+RUN mkdir /etc/service/onyx_peer
+RUN mkdir /etc/service/aeron
+
 RUN apt-get install -y oracle-java8-installer
 
 ADD target/desdemona-0.1.0-SNAPSHOT-standalone.jar /srv/desdemona.jar
 
-ADD script/run-peers.sh /srv/run-peers.sh
+ADD script/run_peers.sh /etc/service/onyx_peer/run
+ADD script/run_aeron.sh /etc/service/aeron/run
+
+EXPOSE 40200/tcp
+EXPOSE 40200/udp
 
-ENTRYPOINT ["/bin/sh", "/srv/run-peers.sh"]
+CMD ["/sbin/my_init"]

README.md

@@ -34,6 +34,45 @@ You can launch a sample job as follows:
 (desdemona.jobs.sample-submit-job/submit-job user/system)
 ```
 
+### Using Docker Compose
+
+Start the cluster:
+
+```
+docker-compose up
+```
+
+Wait until it's all started. It should say this and then wait:
+
+```
+peer_1      | Started peers. Blocking forever.
+```
+
+Make sure you create the Kafka topic:
+
+```
+docker run --rm -it --link desdemona_kafka_1:kafka1 kafka bash -c "\$KAFKA_HOME/bin/kafka-console-producer.sh --topic test1 --broker-list=kafka1:9092"
+```
+
+And the MySQL database:
+
+```
+docker run -it --link desdemona_db_1:mysql --rm mysql sh -c 'exec mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD"'
+```
+
+```
+use logs;
+CREATE TABLE logLines (id int primary key auto_increment, line text);
+```
+
+Now you can submit the job:
+
+```
+ZOOKEEPER=$(echo $DOCKER_HOST|cut -d ':' -f 2|sed "s/\/\///g") lein run -m desdemona.jobs.sample-submit-job
+```
+
+Anything you send to syslog on that Docker host (there's a syslog-ng relay running as a container) will appear in MySQL.
+
 ### Production Mode Peers
 
 First start the Aeron media driver, which should be used in production

docker-compose.yml

@@ -0,0 +1,46 @@
+peer:
+        image: 'desdemona:0.1.0'
+        links:
+          - zookeeper:zk
+          - kafka:kafka
+          - db:db
+          - syslog-ng:syslog-ng
+        environment:
+          ONYX_ID: 1
+          NPEERS: 6
+        privileged: true
+        expose:
+          - "40200"
+          - "40200/udp"
+zookeeper:
+        image: 'wurstmeister/zookeeper'
+        ports:
+          - '2181:2181'
+kafka:
+        image: 'wurstmeister/kafka'
+        environment:
+          KAFKA_BROKER_ID: 1
+          HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"
+        links:
+          - zookeeper:zk
+        ports:
+          - "9092:9092"
+        volumes:
+          - /var/run/docker.sock:/var/run/docker.sock
+db:
+        image: mysql
+        environment:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'true'
+          MYSQL_USER: onyx
+          MYSQL_PASSWORD: onyx
+          MYSQL_DATABASE: logs
+        ports:
+          - "3306:3306"
+syslog-ng:
+    build: script/syslog-ng
+    links:
+        - kafka:kafka
+    volumes:
+        - ./script/syslog-ng/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf
+    ports:
+        - "601:601"

env/dev/desdemona/dev_inputs/sample_input.clj

@@ -1,72 +1,14 @@
 (ns desdemona.dev-inputs.sample-input)
 
-;; Story from http://textfiles.com/stories/advtthum.txt
-;; Warning, I haven't read it. :)
-
 (def lines
-  [{:line "                 THE ADVENTURES OF TOM THUMB"}
+  [{:line "2016-01-29T21:44:01+00:00 192.168.99.1 102 <1> Jan 29 15:44:01 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:01.150100217 -0600 CST"}
+   {:line ""}
+   {:line "2016-01-29T21:44:02+00:00 192.168.99.1 102 <1> Jan 29 15:44:01 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:01.654799006 -0600 CST"}
+   {:line ""}
+   {:line "2016-01-29T21:44:02+00:00 192.168.99.1 102 <1> Jan 29 15:44:02 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:02.159437592 -0600 CST"}
+   {:line ""}
+   {:line "2016-01-29T21:44:03+00:00 192.168.99.1 102 <1> Jan 29 15:44:02 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:02.662554386 -0600 CST"}
+   {:line ""}
+   {:line "2016-01-29T21:44:03+00:00 192.168.99.1 102 <1> Jan 29 15:44:03 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:03.164769137 -0600 CST"}
    {:line ""}
-   {:line "   Once upon a time . . . there lived a giant who had quarrelled with a very "}
-   {:line "greedy wizard over sharing a treasure. After the quarrel, the giant said "}
-   {:line "menacingly to the wizard:"}
-   {:line "   \"I could crush you under my thumb if I wanted to! Now, get out of my "}
-   {:line "sight!\" The wizard hurried away, but from a safe distance, he hurled his "}
-   {:line "terrible revenge."}
-   {:line "   \"Abracadabra! Here I cast this spell! May the son, your wife will shortly "}
-   {:line "give you, never grow any taller than my own thumb!\""}
-   {:line "   After Tom Thumb was born, his parents were at their wits' end. They could "}
-   {:line "never find him, for they could barely see him. They had to speak in whispers "}
-   {:line "for fear of deafening the little boy. Tom Thumb preferred playing with the "}
-   {:line "little garden creatures, to the company of parents so different from himself. "}
-   {:line "He rode piggyback on the snail and danced with the ladybirds. Tiny as he was, "}
-   {:line "he had great fun in the world of little things."}
-   {:line "   But one unlucky day, he went to visit a froggy friend. No sooner had he "}
-   {:line "scrambled onto a leaf than a large pike swallowed him up. But the pike too was"}
-   {:line "fated to come to a very bad end. A little later, he took the bait cast by one "}
-   {:line "of the King's fishermen, and before long, found himself under the cook's knife"}
-   {:line "in the royal kitchens. And great was everyone's surprise when, out of the "}
-   {:line "fish's stomach, stepped Tom Thumb, quite alive and little the worse for his "}
-   {:line "adventure."}
-   {:line "   \"What am I to do with this tiny lad?\" said the cook to himself. Then he had"}
-   {:line "a brainwave. \"He can be a royal pageboy! He's so tiny, I can pop him into the "}
-   {:line "cake I'm making. When he marches across the bridge, sounding the trumpet "}
-   {:line "everyone will gasp in wonder!\" Never had such a marvel been seen at Court. The"}
-   {:line "guests clapped excitedly at the cook's skill and the King himself clapped "}
-   {:line "loudest of all. The King rewarded the clever cook with a bag of gold. Tom "}
-   {:line "Thumb was even luckier. The cook made him a pageboy, and a pageboy he remained,"}
-   {:line "enjoying all the honours of his post."}
-   {:line "   He had a white mouse for a mount, a gold pin for a sword and he was allowed"}
-   {:line "to eat the King's food. In exchange, he marched up and down the table at "}
-   {:line "banquets. He picked his way amongst the plates and glasses amusing the guests"}
-   {:line "with his trumpet."}
-   {:line "   What Tom Thumb didn't know was that he had made an enemy. The cat which, "}
-   {:line "until Tom's arrival, had been the King's pet, was now forgotten. And, vowing "}
-   {:line "to have its revenge on the newcomer, it ambushed Tom in the garden. When Tom "}
-   {:line "saw the cat, he did not run away, as the creature had intended. He whipped out"}
-   {:line "his gold pin and cried to his white mouse mount:"}
-   {:line "   \"Charge! Charge!\" Jabbed by the tiny sword, the cat turned tail and fled. "}
-   {:line "Since brute force was not the way to revenge, the cat decided to use guile. "}
-   {:line "Casually pretending to bump into the King as he walked down the staircase, the"}
-   {:line "cat softly miaowed:"}
-   {:line "   \"Sire! Be on your guard! A plot is being hatched against your life!\" And"}
-   {:line "then he told a dreadful lie. \"Tom Thumb is planning to lace your food with "}
-   {:line "hemlock. I saw him picking the leaves in the garden the other day. heard him "}
-   {:line "say these very words!\""}
-   {:line "   Now, the King had once been kept in bed with very bad tummy pains, after "}
-   {:line "eating too many chernes and he feared the thought of being poisoned, so he "}
-   {:line "sent for Tom Thumb. The cat provided proof of his words by pulling a hemlock "}
-   {:line "leaf from under the white mouse's saddle cloth, where he had hidden it "}
-   {:line "himself."}
-   {:line "   Tom Thumb was so amazed, he was at a loss for words to deny what the cat "}
-   {:line "had said. The King, withiut further ado, had him thrown into prison. And since"}
-   {:line "he was so tiny, they locked him up in a pendulum clock. The hours passed and"}
-   {:line "the days too. Tom's only pastime was swinging back and forth, clinging to the "}
-   {:line "pendulum, util the night when he attracted the attention of a big night moth,"}
-   {:line "fluttering round the room."}
-   {:line "   \"Let me out!\" cried Tom Thumb, tapping on the glass. As it so happens, the "}
-   {:line "moth had only just been set free after being a prisoner in a large box, in "}
-   {:line "which she had taken a nap. So she took pity on Tom Thumb  and released him."}
-   {:line "   'll take you to the Butterfly Kingdom, where everyone's tiny like burself. "}
-   {:line "They'll take care of you there!\" And that is what happened. To this day, if "}
-   {:line "you visit the Butterfly Kingdom, you can ask to see the Butterfly monument "}
-   {:line "that Tom Thumb built after this amazing adventrure."}])
+   {:line "2016-01-29T21:44:04+00:00 192.168.99.1 102 <1> Jan 29 15:44:03 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:03.669196559 -0600 CST"}])

project.clj

@@ -4,7 +4,14 @@
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure "1.8.0"]
-                 [org.onyxplatform/onyx "0.8.2"]
+                 [org.onyxplatform/onyx "0.8.8"]
+                 [org.onyxplatform/onyx-sql "0.8.8.0"]
+                 [mysql/mysql-connector-java "5.1.18"]
+                 [org.onyxplatform/onyx-kafka "0.8.8.0"]
+                 [org.onyxplatform/onyx-seq "0.8.8.0"]
+                 [cheshire "5.5.0"]
+                 [org.clojure/tools.cli "0.3.3"]
+                 [aero "0.1.3"]
                  [org.clojure/core.logic "0.8.10"]]
   :plugins [[lein-cljfmt "0.3.0"]
             [lein-cloverage "1.0.7-SNAPSHOT"]

resources/config.edn

@@ -0,0 +1,24 @@
+{:env-config
+ {:onyx/id #cond {:default #env ONYX_ID
+                  :dev "1"}
+  :onyx.bookkeeper/server? true
+  :onyx.bookkeeper/delete-server-data? true
+  :zookeeper/address #cond {:default #env [ZOOKEEPER "zk:2181"]
+                            :test #env [ZOOKEEPER "127.0.0.1:2181"]}
+  :zookeeper/server? #cond {:default false
+                            :test true}
+  :zookeeper.server/port 2181}
+ :peer-config
+ {:onyx/id                               #cond {:default #env ONYX_ID
+                                                :dev "1"}
+  :zookeeper/address                     #cond {:default #env [ZOOKEEPER "zk:2181"]
+                                                :dev #env [ZOOKEEPER "192.168.99.100:2181"]}
+  :onyx.peer/job-scheduler               :onyx.job-scheduler/greedy
+  :onyx.peer/zookeeper-timeout           60000
+  :onyx.messaging/allow-short-circuit?   true
+  :onyx.messaging/impl                   :aeron
+  ;; Change "localhost" to a resolvable hostname
+  ;; by any node in your cluster.
+  :onyx.messaging/bind-addr             #cond {:default #env [BIND_ADDR "localhost"]}
+  :onyx.messaging/peer-port              40200
+  :onyx.messaging.aeron/embedded-driver? false}}

resources/sample_input.edn

@@ -0,0 +1,21 @@
+[{"venue_visibility" "public", "payment_required" "0", "yes_rsvp_count" 4, "group" {"group_lon" -123.14, "country" "ca", "city" "Vancouver", "group_lat" 49.26, "id" 1734488, "group_photo" {"highres_link" "http://photos4.meetupstatic.com/photos/event/1/d/3/highres_431820467.jpeg", "photo_link" "http://photos4.meetupstatic.com/photos/event/1/d/3/600_431820467.jpeg", "photo_id" 431820467, "thumb_link" "http://photos2.meetupstatic.com/photos/event/1/d/3/thumb_431820467.jpeg"}, "name" "Kitsilano Business Leaders", "state" "BC", "category" {"name" "career/business", "id" 2, "shortname" "career-business"}, "urlname" "KitsilanoBusinessLeaders", "join_mode" "open"}, "id" "227142972", "utc_offset" -28800000, "visibility" "public", "name" "12 Steps How to Build Mobile App to Rock Your Business", "maybe_rsvp_count" 0, "status" "upcoming", "venue" {"country" "ca", "city" "Vancouver", "address_1" "2941 West Broadway", "name" "G&F Financial Group", "lon" -123.170868, "state" "BC", "lat" 49.26437}, "time" 1452783600000, "rsvp_limit" 60, "description" "<p>Do you have a mobile app for your business? Do you have an&nbsp;effective mobile strategy that involves more than just having a mobile-friendly website?</p>\n<p>Following Melanie Haselmayr's Forbes article, we are featuring two local entrepreneurs who will share the 12 steps they took to build a dynamite app for their businesses.&nbsp;<a href=\"http://www.forbes.com/sites/allbusiness/2014/11/17/heres-why-your-business-needs-its-own-mobile-app/\" class=\"linkified\">http://www.forbes.com/sites/allbusiness/2014/11/17/heres-why-your-business-needs-its-own-mobile-app/</a></p>\n<p><br></p>\n<p>&nbsp;Ria Mizera, founder of ClosetRelay,<br></p>\n<p><img src=\"http://photos3.meetupstatic.com/photos/event/7/b/b/b/600_445111675.jpeg\"></p>\n<p><br></p>\n<p> <br></p>\n<p><img src=\"http://photos1.meetupstatic.com/photos/event/7/c/8/3/600_445111875.jpeg\"></p>\n<p>Kristoffer Vik Hansen, co-founder of Spare</p>\n<p> <br></p>\n<p><i>Early risers, come and&nbsp;kickstart your day once a month with insightful discussions on current business challenges.</i></p>\n<p><i>MEETING SCHEDULE</i></p>\n<p><i>6:50 - 7:20 am: Sign in, get your coffee and muffin early. Be ready to develop business trust and camaraderie.&nbsp;</i></p>\n<p><i>7:20 - 7:45 am: Muffin Sponsor (5 min.), Success Circle...15 second insight on who you are and how your business can help others to succeed.&nbsp;</i></p>\n<p><i>7:45 - 8:30 am: Keynote presentation&nbsp;</i></p>\n<p><i>8:30 - 9:00 am: Door prizes; share and learn camaraderie continues&nbsp;</i></p>\n<p><i>DOOR PRIZE donors:&nbsp;</i>Contact Judy if you have a product/service to promote.</p>\n<p><i>MUFFIN SPONSOR:&nbsp;</i>Contact Judy how you can be&nbsp;muffin sponsor of the month and&nbsp;promote your business for 5 minutes to a captive audience.</p>\n<br>\n<br>", "event_url" "http://www.meetup.com/KitsilanoBusinessLeaders/events/227142972/", "mtime" 1450129667481}
+ {"venue_visibility" "members", "payment_required" "0", "yes_rsvp_count" 1, "group" {"group_lon" -111.75, "country" "us", "city" "Gilbert", "group_lat" 33.36, "id" 5391722, "group_photo" {"highres_link" "http://photos4.meetupstatic.com/photos/event/e/2/4/highres_437583620.jpeg", "photo_link" "http://photos4.meetupstatic.com/photos/event/e/2/4/600_437583620.jpeg", "photo_id" 437583620, "thumb_link" "http://photos2.meetupstatic.com/photos/event/e/2/4/thumb_437583620.jpeg"}, "name" "Chandler Gilbert Ladies and Gents Singles Over 45", "state" "AZ", "category" {"name" "parents/family", "id" 25, "shortname" "parents-family"}, "urlname" "meetup-group-HwhOSVfy", "join_mode" "open"}, "id" "227421769", "utc_offset" -25200000, "visibility" "public", "name" "Let's meetup for a walk...", "maybe_rsvp_count" 0, "status" "suggested", "time" 1452524400000, "duration" 5400000, "rsvp_limit" 0, "description" "<p>Let's meetup for some fun and fitness while walking at the Riparian Preserve.</p>", "event_url" "http://www.meetup.com/meetup-group-HwhOSVfy/events/227421769/", "mtime" 1450129667292}
+ {"venue_visibility" "members",
+  "payment_required" "0",
+  "yes_rsvp_count"   1,
+  "group"            {"group_lon"   -118.33, "country" "us", "city" "Los Angeles", "group_lat" 34.1, "id" 511174,
+                      "group_photo" {"highres_link" "http://photos3.meetupstatic.com/photos/event/5/5/1/d/highres_436221789.jpeg",
+                                     "photo_link"   "http://photos3.meetupstatic.com/photos/event/5/5/1/d/600_436221789.jpeg",
+                                     "photo_id"     436221789,
+                                     "thumb_link"   "http://photos3.meetupstatic.com/photos/event/5/5/1/d/thumb_436221789.jpeg"},
+                      "name"        "L.A. Foodies", "state" "CA",
+                      "category"    {"name" "food/drink",
+                                     "id"   10, "shortname" "food-drink"},
+                      "urlname"     "Foodies", "join_mode" "approval"},
+  "id"               "227420766", "utc_offset" -28800000, "visibility" "public",
+  "name"             "$85 five-course tasting menu @ Trois Mec!",
+  "maybe_rsvp_count" 0, "status" "upcoming",
+  "time"             1452304800000, "rsvp_limit" 6,
+  "event_url"        "http://www.meetup.com/Foodies/events/227420766/",
+  "mtime"            1450129671307}
+ :done]

resources/table.sql

@@ -0,0 +1 @@
+CREATE TABLE logLines (id int primary key auto_increment, line text);

script/run_aeron.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo "Setting shared memory for Aeron"
+mount -t tmpfs -o remount,rw,nosuid,nodev,noexec,relatime,size=256M tmpfs /dev/shm
+APP_NAME=$(echo "desdemona" | sed s/"-"/"_"/g)
+
+exec java -cp /srv/desdemona.jar "$APP_NAME.launcher.aeron_media_driver" >>/var/log/aeron.log 2>&1

script/run_peers.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o xtrace
+
+export BIND_ADDR=$(hostname --ip-address)
+export APP_NAME=$(echo "desdemona" | sed s/"-"/"_"/g)
+exec java -cp /srv/desdemona.jar "$APP_NAME.launcher.launch_prod_peers" $NPEERS

script/syslog-ng/Dockerfile

@@ -0,0 +1,24 @@
+FROM debian:latest
+MAINTAINER Andras Mitzki <andras.mitzki@balabit.com>
+
+RUN apt-get update -qq && apt-get install -y \
+    wget
+
+RUN wget -qO - http://download.opensuse.org/repositories/home:/laszlo_budai:/syslog-ng/Debian_8.0/Release.key | apt-key add -
+RUN echo 'deb http://download.opensuse.org/repositories/home:/laszlo_budai:/syslog-ng/Debian_8.0 ./' | tee --append /etc/apt/sources.list.d/syslog-ng-obs.list
+
+RUN apt-get update -qq && apt-get install -y \
+    syslog-ng
+
+ADD openjdk-libjvm.conf /etc/ld.so.conf.d/openjdk-libjvm.conf
+RUN ldconfig
+
+RUN wget -q http://artfiles.org/apache.org/kafka/0.9.0.0/kafka_2.11-0.9.0.0.tgz -O /tmp/kafka_2.11-0.9.0.0.tgz
+RUN tar xfz /tmp/kafka_2.11-0.9.0.0.tgz -C /opt
+RUN ln -s /opt/kafka_2.11-0.9.0.0 /opt/kafka
+
+EXPOSE 514
+EXPOSE 601
+EXPOSE 6514
+
+ENTRYPOINT ["/usr/sbin/syslog-ng", "-F"]

script/syslog-ng/openjdk-libjvm.conf

@@ -0,0 +1 @@
+/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server/