This repository has been archived by the owner on Mar 18, 2019. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #21 from sirsean/kafka-ingestion
Kafka ingestion
- Loading branch information
Showing
40 changed files
with
690 additions
and
615 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
peer: | ||
image: 'desdemona:0.1.0' | ||
links: | ||
- zookeeper:zk | ||
- kafka:kafka | ||
- db:db | ||
- syslog-ng:syslog-ng | ||
environment: | ||
ONYX_ID: 1 | ||
NPEERS: 6 | ||
privileged: true | ||
expose: | ||
- "40200" | ||
- "40200/udp" | ||
zookeeper: | ||
image: 'wurstmeister/zookeeper' | ||
ports: | ||
- '2181:2181' | ||
kafka: | ||
image: 'wurstmeister/kafka' | ||
environment: | ||
KAFKA_BROKER_ID: 1 | ||
HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'" | ||
links: | ||
- zookeeper:zk | ||
ports: | ||
- "9092:9092" | ||
volumes: | ||
- /var/run/docker.sock:/var/run/docker.sock | ||
db: | ||
image: mysql | ||
environment: | ||
MYSQL_ALLOW_EMPTY_PASSWORD: 'true' | ||
MYSQL_USER: onyx | ||
MYSQL_PASSWORD: onyx | ||
MYSQL_DATABASE: logs | ||
ports: | ||
- "3306:3306" | ||
syslog-ng: | ||
build: script/syslog-ng | ||
links: | ||
- kafka:kafka | ||
volumes: | ||
- ./script/syslog-ng/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf | ||
ports: | ||
- "601:601" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,72 +1,14 @@ | ||
(ns desdemona.dev-inputs.sample-input) | ||
|
||
;; Story from http://textfiles.com/stories/advtthum.txt | ||
;; Warning, I haven't read it. :) | ||
|
||
(def lines | ||
[{:line " THE ADVENTURES OF TOM THUMB"} | ||
[{:line "2016-01-29T21:44:01+00:00 192.168.99.1 102 <1> Jan 29 15:44:01 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:01.150100217 -0600 CST"} | ||
{:line ""} | ||
{:line "2016-01-29T21:44:02+00:00 192.168.99.1 102 <1> Jan 29 15:44:01 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:01.654799006 -0600 CST"} | ||
{:line ""} | ||
{:line "2016-01-29T21:44:02+00:00 192.168.99.1 102 <1> Jan 29 15:44:02 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:02.159437592 -0600 CST"} | ||
{:line ""} | ||
{:line "2016-01-29T21:44:03+00:00 192.168.99.1 102 <1> Jan 29 15:44:02 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:02.662554386 -0600 CST"} | ||
{:line ""} | ||
{:line "2016-01-29T21:44:03+00:00 192.168.99.1 102 <1> Jan 29 15:44:03 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:03.164769137 -0600 CST"} | ||
{:line ""} | ||
{:line " Once upon a time . . . there lived a giant who had quarrelled with a very "} | ||
{:line "greedy wizard over sharing a treasure. After the quarrel, the giant said "} | ||
{:line "menacingly to the wizard:"} | ||
{:line " \"I could crush you under my thumb if I wanted to! Now, get out of my "} | ||
{:line "sight!\" The wizard hurried away, but from a safe distance, he hurled his "} | ||
{:line "terrible revenge."} | ||
{:line " \"Abracadabra! Here I cast this spell! May the son, your wife will shortly "} | ||
{:line "give you, never grow any taller than my own thumb!\""} | ||
{:line " After Tom Thumb was born, his parents were at their wits' end. They could "} | ||
{:line "never find him, for they could barely see him. They had to speak in whispers "} | ||
{:line "for fear of deafening the little boy. Tom Thumb preferred playing with the "} | ||
{:line "little garden creatures, to the company of parents so different from himself. "} | ||
{:line "He rode piggyback on the snail and danced with the ladybirds. Tiny as he was, "} | ||
{:line "he had great fun in the world of little things."} | ||
{:line " But one unlucky day, he went to visit a froggy friend. No sooner had he "} | ||
{:line "scrambled onto a leaf than a large pike swallowed him up. But the pike too was"} | ||
{:line "fated to come to a very bad end. A little later, he took the bait cast by one "} | ||
{:line "of the King's fishermen, and before long, found himself under the cook's knife"} | ||
{:line "in the royal kitchens. And great was everyone's surprise when, out of the "} | ||
{:line "fish's stomach, stepped Tom Thumb, quite alive and little the worse for his "} | ||
{:line "adventure."} | ||
{:line " \"What am I to do with this tiny lad?\" said the cook to himself. Then he had"} | ||
{:line "a brainwave. \"He can be a royal pageboy! He's so tiny, I can pop him into the "} | ||
{:line "cake I'm making. When he marches across the bridge, sounding the trumpet "} | ||
{:line "everyone will gasp in wonder!\" Never had such a marvel been seen at Court. The"} | ||
{:line "guests clapped excitedly at the cook's skill and the King himself clapped "} | ||
{:line "loudest of all. The King rewarded the clever cook with a bag of gold. Tom "} | ||
{:line "Thumb was even luckier. The cook made him a pageboy, and a pageboy he remained,"} | ||
{:line "enjoying all the honours of his post."} | ||
{:line " He had a white mouse for a mount, a gold pin for a sword and he was allowed"} | ||
{:line "to eat the King's food. In exchange, he marched up and down the table at "} | ||
{:line "banquets. He picked his way amongst the plates and glasses amusing the guests"} | ||
{:line "with his trumpet."} | ||
{:line " What Tom Thumb didn't know was that he had made an enemy. The cat which, "} | ||
{:line "until Tom's arrival, had been the King's pet, was now forgotten. And, vowing "} | ||
{:line "to have its revenge on the newcomer, it ambushed Tom in the garden. When Tom "} | ||
{:line "saw the cat, he did not run away, as the creature had intended. He whipped out"} | ||
{:line "his gold pin and cried to his white mouse mount:"} | ||
{:line " \"Charge! Charge!\" Jabbed by the tiny sword, the cat turned tail and fled. "} | ||
{:line "Since brute force was not the way to revenge, the cat decided to use guile. "} | ||
{:line "Casually pretending to bump into the King as he walked down the staircase, the"} | ||
{:line "cat softly miaowed:"} | ||
{:line " \"Sire! Be on your guard! A plot is being hatched against your life!\" And"} | ||
{:line "then he told a dreadful lie. \"Tom Thumb is planning to lace your food with "} | ||
{:line "hemlock. I saw him picking the leaves in the garden the other day. heard him "} | ||
{:line "say these very words!\""} | ||
{:line " Now, the King had once been kept in bed with very bad tummy pains, after "} | ||
{:line "eating too many chernes and he feared the thought of being poisoned, so he "} | ||
{:line "sent for Tom Thumb. The cat provided proof of his words by pulling a hemlock "} | ||
{:line "leaf from under the white mouse's saddle cloth, where he had hidden it "} | ||
{:line "himself."} | ||
{:line " Tom Thumb was so amazed, he was at a loss for words to deny what the cat "} | ||
{:line "had said. The King, withiut further ado, had him thrown into prison. And since"} | ||
{:line "he was so tiny, they locked him up in a pendulum clock. The hours passed and"} | ||
{:line "the days too. Tom's only pastime was swinging back and forth, clinging to the "} | ||
{:line "pendulum, util the night when he attracted the attention of a big night moth,"} | ||
{:line "fluttering round the room."} | ||
{:line " \"Let me out!\" cried Tom Thumb, tapping on the glass. As it so happens, the "} | ||
{:line "moth had only just been set free after being a prisoner in a large box, in "} | ||
{:line "which she had taken a nap. So she took pity on Tom Thumb and released him."} | ||
{:line " 'll take you to the Butterfly Kingdom, where everyone's tiny like burself. "} | ||
{:line "They'll take care of you there!\" And that is what happened. To this day, if "} | ||
{:line "you visit the Butterfly Kingdom, you can ask to see the Butterfly monument "} | ||
{:line "that Tom Thumb built after this amazing adventrure."}]) | ||
{:line "2016-01-29T21:44:04+00:00 192.168.99.1 102 <1> Jan 29 15:44:03 lips.local sean[68673]: hello ernie, it's 2016-01-29 15:44:03.669196559 -0600 CST"}]) |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
{:env-config | ||
{:onyx/id #cond {:default #env ONYX_ID | ||
:dev "1"} | ||
:onyx.bookkeeper/server? true | ||
:onyx.bookkeeper/delete-server-data? true | ||
:zookeeper/address #cond {:default #env [ZOOKEEPER "zk:2181"] | ||
:test #env [ZOOKEEPER "127.0.0.1:2181"]} | ||
:zookeeper/server? #cond {:default false | ||
:test true} | ||
:zookeeper.server/port 2181} | ||
:peer-config | ||
{:onyx/id #cond {:default #env ONYX_ID | ||
:dev "1"} | ||
:zookeeper/address #cond {:default #env [ZOOKEEPER "zk:2181"] | ||
:dev #env [ZOOKEEPER "192.168.99.100:2181"]} | ||
:onyx.peer/job-scheduler :onyx.job-scheduler/greedy | ||
:onyx.peer/zookeeper-timeout 60000 | ||
:onyx.messaging/allow-short-circuit? true | ||
:onyx.messaging/impl :aeron | ||
;; Change "localhost" to a resolvable hostname | ||
;; by any node in your cluster. | ||
:onyx.messaging/bind-addr #cond {:default #env [BIND_ADDR "localhost"]} | ||
:onyx.messaging/peer-port 40200 | ||
:onyx.messaging.aeron/embedded-driver? false}} |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
[{"venue_visibility" "public", "payment_required" "0", "yes_rsvp_count" 4, "group" {"group_lon" -123.14, "country" "ca", "city" "Vancouver", "group_lat" 49.26, "id" 1734488, "group_photo" {"highres_link" "http://photos4.meetupstatic.com/photos/event/1/d/3/highres_431820467.jpeg", "photo_link" "http://photos4.meetupstatic.com/photos/event/1/d/3/600_431820467.jpeg", "photo_id" 431820467, "thumb_link" "http://photos2.meetupstatic.com/photos/event/1/d/3/thumb_431820467.jpeg"}, "name" "Kitsilano Business Leaders", "state" "BC", "category" {"name" "career/business", "id" 2, "shortname" "career-business"}, "urlname" "KitsilanoBusinessLeaders", "join_mode" "open"}, "id" "227142972", "utc_offset" -28800000, "visibility" "public", "name" "12 Steps How to Build Mobile App to Rock Your Business", "maybe_rsvp_count" 0, "status" "upcoming", "venue" {"country" "ca", "city" "Vancouver", "address_1" "2941 West Broadway", "name" "G&F Financial Group", "lon" -123.170868, "state" "BC", "lat" 49.26437}, "time" 1452783600000, "rsvp_limit" 60, "description" "<p>Do you have a mobile app for your business? Do you have an effective mobile strategy that involves more than just having a mobile-friendly website?</p>\n<p>Following Melanie Haselmayr's Forbes article, we are featuring two local entrepreneurs who will share the 12 steps they took to build a dynamite app for their businesses. <a href=\"http://www.forbes.com/sites/allbusiness/2014/11/17/heres-why-your-business-needs-its-own-mobile-app/\" class=\"linkified\">http://www.forbes.com/sites/allbusiness/2014/11/17/heres-why-your-business-needs-its-own-mobile-app/</a></p>\n<p><br></p>\n<p> Ria Mizera, founder of ClosetRelay,<br></p>\n<p><img src=\"http://photos3.meetupstatic.com/photos/event/7/b/b/b/600_445111675.jpeg\"></p>\n<p><br></p>\n<p> <br></p>\n<p><img src=\"http://photos1.meetupstatic.com/photos/event/7/c/8/3/600_445111875.jpeg\"></p>\n<p>Kristoffer Vik Hansen, co-founder of Spare</p>\n<p> <br></p>\n<p><i>Early risers, come and kickstart your day once a month with insightful discussions on current business challenges.</i></p>\n<p><i>MEETING SCHEDULE</i></p>\n<p><i>6:50 - 7:20 am: Sign in, get your coffee and muffin early. Be ready to develop business trust and camaraderie. </i></p>\n<p><i>7:20 - 7:45 am: Muffin Sponsor (5 min.), Success Circle...15 second insight on who you are and how your business can help others to succeed. </i></p>\n<p><i>7:45 - 8:30 am: Keynote presentation </i></p>\n<p><i>8:30 - 9:00 am: Door prizes; share and learn camaraderie continues </i></p>\n<p><i>DOOR PRIZE donors: </i>Contact Judy if you have a product/service to promote.</p>\n<p><i>MUFFIN SPONSOR: </i>Contact Judy how you can be muffin sponsor of the month and promote your business for 5 minutes to a captive audience.</p>\n<br>\n<br>", "event_url" "http://www.meetup.com/KitsilanoBusinessLeaders/events/227142972/", "mtime" 1450129667481} | ||
{"venue_visibility" "members", "payment_required" "0", "yes_rsvp_count" 1, "group" {"group_lon" -111.75, "country" "us", "city" "Gilbert", "group_lat" 33.36, "id" 5391722, "group_photo" {"highres_link" "http://photos4.meetupstatic.com/photos/event/e/2/4/highres_437583620.jpeg", "photo_link" "http://photos4.meetupstatic.com/photos/event/e/2/4/600_437583620.jpeg", "photo_id" 437583620, "thumb_link" "http://photos2.meetupstatic.com/photos/event/e/2/4/thumb_437583620.jpeg"}, "name" "Chandler Gilbert Ladies and Gents Singles Over 45", "state" "AZ", "category" {"name" "parents/family", "id" 25, "shortname" "parents-family"}, "urlname" "meetup-group-HwhOSVfy", "join_mode" "open"}, "id" "227421769", "utc_offset" -25200000, "visibility" "public", "name" "Let's meetup for a walk...", "maybe_rsvp_count" 0, "status" "suggested", "time" 1452524400000, "duration" 5400000, "rsvp_limit" 0, "description" "<p>Let's meetup for some fun and fitness while walking at the Riparian Preserve.</p>", "event_url" "http://www.meetup.com/meetup-group-HwhOSVfy/events/227421769/", "mtime" 1450129667292} | ||
{"venue_visibility" "members", | ||
"payment_required" "0", | ||
"yes_rsvp_count" 1, | ||
"group" {"group_lon" -118.33, "country" "us", "city" "Los Angeles", "group_lat" 34.1, "id" 511174, | ||
"group_photo" {"highres_link" "http://photos3.meetupstatic.com/photos/event/5/5/1/d/highres_436221789.jpeg", | ||
"photo_link" "http://photos3.meetupstatic.com/photos/event/5/5/1/d/600_436221789.jpeg", | ||
"photo_id" 436221789, | ||
"thumb_link" "http://photos3.meetupstatic.com/photos/event/5/5/1/d/thumb_436221789.jpeg"}, | ||
"name" "L.A. Foodies", "state" "CA", | ||
"category" {"name" "food/drink", | ||
"id" 10, "shortname" "food-drink"}, | ||
"urlname" "Foodies", "join_mode" "approval"}, | ||
"id" "227420766", "utc_offset" -28800000, "visibility" "public", | ||
"name" "$85 five-course tasting menu @ Trois Mec!", | ||
"maybe_rsvp_count" 0, "status" "upcoming", | ||
"time" 1452304800000, "rsvp_limit" 6, | ||
"event_url" "http://www.meetup.com/Foodies/events/227420766/", | ||
"mtime" 1450129671307} | ||
:done] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
CREATE TABLE logLines (id int primary key auto_increment, line text); |
Empty file.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
#!/bin/sh | ||
|
||
echo "Setting shared memory for Aeron" | ||
mount -t tmpfs -o remount,rw,nosuid,nodev,noexec,relatime,size=256M tmpfs /dev/shm | ||
APP_NAME=$(echo "desdemona" | sed s/"-"/"_"/g) | ||
|
||
exec java -cp /srv/desdemona.jar "$APP_NAME.launcher.aeron_media_driver" >>/var/log/aeron.log 2>&1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/usr/bin/env bash | ||
set -o errexit | ||
set -o nounset | ||
set -o xtrace | ||
|
||
export BIND_ADDR=$(hostname --ip-address) | ||
export APP_NAME=$(echo "desdemona" | sed s/"-"/"_"/g) | ||
exec java -cp /srv/desdemona.jar "$APP_NAME.launcher.launch_prod_peers" $NPEERS |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
FROM debian:latest | ||
MAINTAINER Andras Mitzki <andras.mitzki@balabit.com> | ||
|
||
RUN apt-get update -qq && apt-get install -y \ | ||
wget | ||
|
||
RUN wget -qO - http://download.opensuse.org/repositories/home:/laszlo_budai:/syslog-ng/Debian_8.0/Release.key | apt-key add - | ||
RUN echo 'deb http://download.opensuse.org/repositories/home:/laszlo_budai:/syslog-ng/Debian_8.0 ./' | tee --append /etc/apt/sources.list.d/syslog-ng-obs.list | ||
|
||
RUN apt-get update -qq && apt-get install -y \ | ||
syslog-ng | ||
|
||
ADD openjdk-libjvm.conf /etc/ld.so.conf.d/openjdk-libjvm.conf | ||
RUN ldconfig | ||
|
||
RUN wget -q http://artfiles.org/apache.org/kafka/0.9.0.0/kafka_2.11-0.9.0.0.tgz -O /tmp/kafka_2.11-0.9.0.0.tgz | ||
RUN tar xfz /tmp/kafka_2.11-0.9.0.0.tgz -C /opt | ||
RUN ln -s /opt/kafka_2.11-0.9.0.0 /opt/kafka | ||
|
||
EXPOSE 514 | ||
EXPOSE 601 | ||
EXPOSE 6514 | ||
|
||
ENTRYPOINT ["/usr/sbin/syslog-ng", "-F"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server/ |
Oops, something went wrong.