Permalink
Browse files

Build grok and logstash as .deb on Ubuntu 11.10

  • Loading branch information...
0 parents commit d5ce23268de1e915330c3d1ee55a9f5663b32489 @jbraeuer committed Feb 4, 2012
Showing with 398 additions and 0 deletions.
  1. +3 −0 .gitignore
  2. +3 −0 .gitmodules
  3. +10 −0 Readme.markdown
  4. +280 −0 grok/Makefile
  5. +5 −0 grok/debian/changelog
  6. +1 −0 grok/debian/compat
  7. +12 −0 grok/debian/control
  8. +1 −0 grok/debian/copyright
  9. +3 −0 grok/debian/rules
  10. +1 −0 grok/debian/source/format
  11. +4 −0 grok/grok_version.h
  12. +74 −0 package.sh
  13. +1 −0 upstream/grok
3 .gitignore
@@ -0,0 +1,3 @@
+*~
+download
+work
3 .gitmodules
@@ -0,0 +1,3 @@
+[submodule "upstream/grok"]
+ path = upstream/grok
+ url = git://github.com/jordansissel/grok.git
10 Readme.markdown
@@ -0,0 +1,10 @@
+Running logstash from .debs on Ubuntu 11.10
+
+# Install notes
+
+## Install RabbitMQ
+aptitude install rabbitmq-server rabbitmq-stomp
+rabbitmqctl add_user logstash verysecret
+
+# Install ElasticSearch
+aptitude install elasticsearch
280 grok/Makefile
@@ -0,0 +1,280 @@
+PACKAGE=grok
+
+PLATFORM=$(shell (uname -o 2> /dev/null || uname -s) | tr -d "/" 2> /dev/null)
+FLEX?=flex
+
+FORCE_FLEX?=0
+
+ifeq ($(PLATFORM), FreeBSD)
+PREFIX?=/usr/local
+else
+PREFIX?=/usr
+endif
+
+ifeq ($(PLATFORM), Darwin)
+LIBSUFFIX=dylib
+else
+LIBSUFFIX=so
+endif
+
+# On FreeBSD, you may want to set GPERF=/usr/local/bin/gperf since
+# the base system gperf is too old.
+ifeq ($(PLATFORM), FreeBSD)
+GPERF?=/usr/local/bin/gperf
+else
+GPERF?=/usr/bin/gperf
+endif
+
+# For linux, we need libdl for dlopen()
+# On FreeBSD, comment this line out.
+ifeq ($(PLATFORM), GNULinux)
+LDFLAGS+=-ldl
+endif
+
+# #############################################
+# You probably don't need to make changes below
+
+BASE?=.
+MAJOR=$(shell sh $(BASE)/version.sh --major)
+VERSION=$(shell sh $(BASE)/version.sh)
+
+#CFLAGS+=-g
+#LDFLAGS+=-g
+
+CFLAGS+=-pipe -fPIC -I. -O2
+LDFLAGS+=-lpcre -levent -rdynamic -ltokyocabinet
+
+LIBSUFFIX=$(shell sh $(BASE)/platform.sh libsuffix)
+VERLIBSUFFIX=$(shell sh $(BASE)/platform.sh libsuffix $(MAJOR))
+DYNLIBFLAG=$(shell sh $(BASE)/platform.sh dynlibflag)
+LIBNAMEFLAG=$(shell sh $(BASE)/platform.sh libnameflag $(MAJOR) $(INSTALLLIB))
+
+# Sane includes
+CFLAGS+=-I/usr/local/include
+LDFLAGS+=-L/usr/local/lib
+
+# Platform so we know what to dlopen
+CFLAGS+=-DPLATFORM_$(PLATFORM)
+# Uncomment to totally disable logging features
+#CFLAGS+=-DNOLOGGING
+
+EXTRA_CFLAGS?=
+EXTRA_LDFLAGS?=
+CFLAGS+=$(EXTRA_CFLAGS)
+LDFLAGS+=$(EXTRA_LDFLAGS)
+
+### End of user-servicable configuration
+
+CLEANGEN=filters.c grok_matchconf_macro.c *.yy.c *.tab.c *.tab.h
+CLEANOBJ=*.o *_xdr.[ch] *.so
+CLEANBIN=main grokre grok conftest grok_program
+CLEANVER=VERSION grok_version.h grok.spec
+
+GROKOBJ=grok.o grokre.o grok_capture.o grok_pattern.o stringhelper.o \
+ predicates.o grok_capture_xdr.o grok_match.o grok_logging.o \
+ grok_program.o grok_input.o grok_matchconf.o libc_helper.o \
+ grok_matchconf_macro.o filters.o grok_discover.o
+GROKPROGOBJ=grok_input.o grok_program.o grok_matchconf.o $(GROKOBJ)
+
+GROKHEADER=grok.h grok_pattern.h grok_capture.h grok_capture_xdr.h \
+ grok_match.h grok_logging.h grok_discover.h
+
+# grok_version.h is generated by make.
+GROKHEADER+=grok_version.h
+
+.PHONY: all
+all: grok discogrok libgrok.$(LIBSUFFIX) libgrok.$(VERLIBSUFFIX)
+
+.PHONY: package create-package test-package update-version
+package:
+ $(MAKE) $(MAKEFLAGS) create-package
+ $(MAKE) $(MAKEFLAGS) test-package
+
+install: libgrok.$(LIBSUFFIX) grok discogrok $(GROKHEADER)
+ install -d $(DESTDIR)$(PREFIX)/bin
+ install -d $(DESTDIR)$(PREFIX)/lib
+ install -d $(DESTDIR)$(PREFIX)/include
+ install -m 755 grok $(DESTDIR)$(PREFIX)/bin
+ install -m 755 discogrok $(DESTDIR)$(PREFIX)/bin
+ install -m 644 libgrok.$(LIBSUFFIX) $(DESTDIR)$(PREFIX)/lib
+ ln -s libgrok.$(LIBSUFFIX) $(DESTDIR)$(PREFIX)/lib/libgrok.$(LIBSUFFIX).1
+ for header in $(GROKHEADER); do \
+ install -m 644 $$header $(DESTDIR)$(PREFIX)/include; \
+ done
+ install -d $(DESTDIR)$(PREFIX)/share/grok
+ install -d $(DESTDIR)$(PREFIX)/share/grok/patterns
+ install patterns/base $(DESTDIR)$(PREFIX)/share/grok/patterns/
+
+uninstall:
+ rm -f $(DESTDIR)$(PREFIX)/bin/grok
+ rm -f $(DESTDIR)$(PREFIX)/bin/discogrok
+ rm -f $(DESTDIR)$(PREFIX)/lib/libgrok.$(LIBSUFFIX)
+ for header in $(GROKHEADER); do \
+ rm -f $(DESTDIR)$(PREFIX)/include/$$header; \
+ done
+ rm -f $(DESTDIR)$(PREFIX)/share/grok/patterns/*
+
+pre-create-package:
+ rm -f VERSION grok_version.h
+
+create-package: pre-create-package
+ $(MAKE) VERSION grok_version.h grok.spec
+ PACKAGE=$(PACKAGE) sh package.sh $(VERSION)
+
+test-package:
+ PKGVER=$(PACKAGE)-$(VERSION); \
+ tar -C /tmp -zxf $${PKGVER}.tar.gz; \
+ echo "Running C tests..." && $(MAKE) -C /tmp/$${PKGVER}/test test-c
+
+.PHONY: clean
+clean: cleanobj cleanbin
+
+# reallyreallyclean also purges generated files
+# we don't clean generated files in 'clean' target
+# because some systems don't have the tools to regenerate
+# the data, such as FreeBSD which has the wrong flavor
+# of flex (not gnu flex)
+.PHONY: reallyreallyclean
+reallyreallyclean: reallyclean cleangen
+
+.PHONY: reallyclean
+reallyclean: clean cleanver
+
+.PHONY: cleanobj
+cleanobj:
+ rm -f $(CLEANOBJ)
+
+.PHONY: cleanbin
+cleanbin:
+ rm -f $(CLEANBIN)
+
+.PHONY: cleangen
+cleangen:
+ rm -f $(CLEANGEN)
+
+.PHONY: cleanver
+cleanver:
+ rm -f $(CLEANVER)
+
+#.PHONY: test
+#test:
+ #$(MAKE) -C test test
+
+# Binary creation
+grok: LDFLAGS+=-levent
+grok: $(GROKOBJ) conf.tab.o conf.yy.o main.o grok_config.o
+ $(CC) $^ -o $@ $(LDFLAGS)
+
+discogrok: $(GROKOBJ) discover_main.o
+ $(CC) $^ -o $@ $(LDFLAGS)
+
+libgrok.$(LIBSUFFIX):
+libgrok.$(LIBSUFFIX): $(GROKOBJ)
+ $(CC) -fPIC $(DYNLIBFLAG) $(LIBNAMEFLAG) $^ -o $@ $(LDFLAGS)
+
+libgrok.$(VERLIBSUFFIX): libgrok.$(LIBSUFFIX);
+ ln -s $< $@
+
+# File dependencies
+# generated with:
+# for i in *.c; do grep '#include "' $i | fex '"2' | xargs | sed -e "s/^/$i: /"; done
+grok.h: grok_version.h
+grok.c: grok.h
+grok_capture.c: grok.h grok_capture.h grok_capture_xdr.h
+grok_capture_xdr.c: grok_capture.h
+grok_config.c: grok_input.h grok_config.h grok_matchconf.h grok_logging.h
+grok_input.c: grok.h grok_program.h grok_input.h grok_matchconf.h grok_logging.h libc_helper.h
+grok_logging.c: grok.h
+grok_match.c: grok.h
+grok_matchconf.c: grok.h grok_matchconf.h grok_matchconf_macro.h grok_logging.h libc_helper.h filters.h stringhelper.h
+grok_pattern.c: grok.h grok_pattern.h
+grok_program.c: grok.h grok_program.h grok_input.h grok_matchconf.h
+grokre.c: grok.h predicates.h stringhelper.h grok_version.h
+libc_helper.c: libc_helper.h
+main.c: grok.h grok_program.h grok_config.h conf.tab.h
+predicates.c: grok_logging.h predicates.h
+stringhelper.c: stringhelper.h
+filters.h: grok.h
+grok.h: grok_logging.h grok_pattern.h grok_capture.h grok_match.h
+grok_capture.h: grok_capture_xdr.h
+grok_config.h: grok_program.h
+grok_input.h: grok_program.h
+grok_match.h: grok_capture_xdr.h
+grok_matchconf.h: grok.h grok_input.h grok_program.h
+predicates.h: grok.h
+
+
+# Output generation
+grok_capture_xdr.o: grok_capture_xdr.c grok_capture_xdr.h
+grok_capture_xdr.c: grok_capture.x
+ [ -f $@ ] && rm $@ || true
+ rpcgen -c $< -o $@
+grok_capture_xdr.h: grok_capture.x
+ [ -f $@ ] && rm $@ || true
+ rpcgen -h $< -o $@
+
+%.c: %.gperf
+ @if $(GPERF) --version | head -1 | egrep -v '3\.[0-9]+\.[0-9]+' ; then \
+ echo "We require gperf version >= 3.0.3" ; \
+ exit 1; \
+ fi
+ $(GPERF) $< > $@
+
+conf.tab.c conf.tab.h: conf.y
+ bison -d $<
+
+conf.yy.c: conf.lex conf.tab.h
+ @if $(FLEX) --version | grep '^flex version' ; then \
+ if [ "$(FORCE_FLEX)" -eq 1 ] ; then \
+ echo "Bad version of flex detected, but FORCE_FLEX is set, trying anyway."; \
+ exit 0; \
+ fi; \
+ echo "Fatal - cannot build"; \
+ echo "You need GNU flex. You seem to have BSD flex?"; \
+ strings `which flex` | grep Regents; \
+ echo "If you want to try your flex, anyway, set FORCE_FLEX=1"; \
+ exit 1; \
+ fi
+ $(FLEX) -o $@ $<
+
+.c.o:
+ $(CC) -c $(CFLAGS) $< -o $@
+
+%.1: %.pod
+ pod2man -c "" -r "" $< $@
+
+grok_version.h:
+ sh $(BASE)/version.sh --header > $@
+
+VERSION:
+ sh $(BASE)/version.sh --shell > $@
+
+grok.spec: grok.spec.template VERSION
+ . ./VERSION;
+ sed -e "s/^Version: .*/Version: $(VERSION)/" grok.spec.template > grok.spec
+
+
+.PHONY: docs
+docs:
+ doxygen
+
+.PHONY: package-debian
+package-debian: debian
+ CFLAGS="$(CFLAGS)" debuild -uc -us
+
+package-debian-clean:
+ rm -r debian || true
+
+debian:
+ dh_make -s -n -c bsd -e $$USER -p grok_$(VERSION) < /dev/null
+ sed -i -e "s/Build-Depends:.*/&, bison, ctags, flex, gperf, libevent-dev, libpcre3-dev, libtokyocabinet-dev/" debian/control
+ sed -i -e "s/Depends:.*/&, libevent-1.4-2 (>= 1.3), libtokyocabinet8 (>= 1.4.9), libpcre3 (>= 7.6)/" debian/control
+ sed -i -e "s/^Description:.*/Description: A powerful pattern-matching and reacting tool./" debian/control
+ sed -i -e "/^Description/,$$ { /^ *$$/d }" debian/control
+ echo '#!/bin/sh' > debian/postinst
+ echo '[ "$$1" = "configure" ] && ldconfig' >> debian/postinst
+ echo 'exit 0' >> debian/postinst
+ echo '#!/bin/sh' > debian/postrm
+ echo '[ "$$1" = "remove" ] && ldconfig' >> debian/postrm
+ echo 'exit 0' >> debian/postrm
+ chmod 755 debian/postinst debian/postrm
5 grok/debian/changelog
@@ -0,0 +1,5 @@
+grok (1.20120204.1) UNRELEASED; urgency=low
+
+ * Initial release.
+
+ -- Jens Braeuer <braeuer.jens@googlemail.com> Thu, 04 Feb 2012 09:46:45 +0100
1 grok/debian/compat
@@ -0,0 +1 @@
+8
12 grok/debian/control
@@ -0,0 +1,12 @@
+Source: grok
+Maintainer: Jens Braeuer <braeuer.jens@googlemail.com>
+Section: misc
+Priority: optional
+Standards-Version: 3.9.2
+Build-Depends: debhelper (>= 8), libtokyocabinet-dev, libevent-dev, gperf, libz-dev
+
+Package: grok
+Architecture: any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Description: Grok
+ Parsing library
1 grok/debian/copyright
@@ -0,0 +1 @@
+unknown
3 grok/debian/rules
@@ -0,0 +1,3 @@
+#!/usr/bin/make -f
+%:
+ dh $@
1 grok/debian/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
4 grok/grok_version.h
@@ -0,0 +1,4 @@
+#ifndef _VERSION_H_
+#define _VERSION_H_
+static const char *GROK_VERSION = "1.20120204.1";
+#endif /* ifndef _VERSION_H */
74 package.sh
@@ -0,0 +1,74 @@
+#! /bin/bash
+
+log() {
+ echo "$@"
+}
+
+clean() {
+ log "Clean workdir"
+ rm -rf "$work"
+}
+
+get() {
+ log "Get source"
+ mkdir -p "$download"
+
+ #
+ # Download binary version of logstash
+ #
+ cd "$download"
+ file="logstash-1.1.0-monolithic.jar"
+ url="http://semicomplete.com/files/logstash/logstash-1.1.0-monolithic.jar"
+ [ -e "$file" ] || wget --output-document "$file" "$url"
+
+ #
+ # Fake a upstream tar.gz release by using git-archive on the git-checkout
+ # Dependency to grok.git is handled via git submodules
+ #
+ version=$(cat "$base/grok/grok_version.h" | grep GROK_VERSION | awk '{ print $6 }' | tr -d '";')
+ file="$download/grok_$version.orig.tar.gz"
+ if [ ! -e "$file" ]; then
+ cd "$upstream/grok"
+ git archive --format=tar --prefix="grok-$version/" HEAD | gzip > "$file"
+ fi
+}
+
+package_logstash() {
+ log "Package logstash"
+
+ mkdir -p "$work/logstash/usr/share/logstash/"
+ cp "$download/logstash-1.1.0-monolithic.jar" "$work/logstash/usr/share/logstash/"
+
+ cd "$work"
+ fpm -s dir -t deb --name logstash --version 1.1.0 --depends 'openjdk-6-jdk' --depends "grok" -a all -C "$work/logstash" .
+}
+
+
+package_grok() {
+ log "Package grok"
+ mkdir -p "$work"
+
+ version=$(cat "$base/grok/grok_version.h" | grep GROK_VERSION | awk '{ print $6 }' | tr -d '";')
+ file="$download/grok_$version.orig.tar.gz"
+
+ cd "$work"
+ cp "$file" "$work"
+ tar xfz "$file"
+ cp -r $base/grok/* "$work/grok-$version/"
+
+ cd "$work/grok-$version/"
+ debuild -us -uc -F
+}
+
+base=$(dirname $(readlink -f "$0"))
+work="$base/work"
+download="$base/download"
+upstream="$base/upstream"
+
+set -e
+set -x
+
+clean
+get
+package_logstash
+package_grok
1 upstream/grok
@@ -0,0 +1 @@
+Subproject commit 11942705515393ef12a5ed438abff0fe7b3ca232

0 comments on commit d5ce232

Please sign in to comment.