Browse files

Create trunk from releases/scribe-2.0

  • Loading branch information...
0 parents commit a267fd882c2c3b151ffd6f7467b325375376a313 Anthony Giardullo committed Oct 23, 2008
Showing with 8,091 additions and 0 deletions.
  1. +177 −0 LICENSE
  2. +12 −0 Makefile.am
  3. +151 −0 README.BUILD
  4. +253 −0 acinclude.m4
  5. +198 −0 aclocal/ax_boost_base.m4
  6. +110 −0 aclocal/ax_boost_filesystem.m4
  7. +118 −0 aclocal/ax_boost_system.m4
  8. +7 −0 bootstrap.sh
  9. +102 −0 configure.ac
  10. +172 −0 examples/README
  11. +57 −0 examples/example1.conf
  12. +108 −0 examples/example2central.conf
  13. +59 −0 examples/example2client.conf
  14. +60 −0 examples/scribe_cat
  15. +46 −0 examples/scribe_ctrl
  16. +2 −0 global_footer.mk
  17. +19 −0 global_header.mk
  18. +39 −0 if/scribe.thrift
  19. +22 −0 lib/py/Makefile.am
  20. +1 −0 lib/py/scribe/__init__.py
  21. +9 −0 lib/py/scribe/constants.py
  22. +60 −0 lib/py/scribe/scribe-remote
  23. +157 −0 lib/py/scribe/scribe.py
  24. +65 −0 lib/py/scribe/ttypes.py
  25. +7 −0 lib/py/setup.py
  26. +127 −0 src/Makefile.am
  27. +74 −0 src/common.h
  28. +186 −0 src/conf.cpp
  29. +74 −0 src/conf.h
  30. +309 −0 src/conn_pool.cpp
  31. +96 −0 src/conn_pool.h
  32. +78 −0 src/env_default.h
  33. +240 −0 src/file.cpp
  34. +89 −0 src/file.h
  35. +710 −0 src/scribe_server.cpp
  36. +92 −0 src/scribe_server.h
  37. +2,110 −0 src/store.cpp
  38. +531 −0 src/store.h
  39. +343 −0 src/store_queue.cpp
  40. +119 −0 src/store_queue.h
  41. +45 −0 test/many_connections.php
  42. +42 −0 test/resultChecker/makefile
  43. +96 −0 test/resultChecker/resultChecker.cpp
  44. +124 −0 test/scribe.conf.test
  45. +76 −0 test/scribe.conf.test2
  46. +23 −0 test/shutdown.php
  47. +23 −0 test/simple_test.php
  48. +23 −0 test/strange_input.php
  49. +31 −0 test/stress.php
  50. +316 −0 test/tests.php
  51. +103 −0 test/tests.txt
177 LICENSE
@@ -0,0 +1,177 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
12 Makefile.am
@@ -0,0 +1,12 @@
+@GLOBAL_HEADER_MK@
+
+@PRODUCT_MK@
+
+SUBDIRS = . src lib/py
+
+BUILT_SOURCES =
+
+
+clean-local: clean-common
+
+@GLOBAL_FOOTER_MK@
151 README.BUILD
@@ -0,0 +1,151 @@
+Scribe
+
+Last Modified: 2008-Sep-18
+
+
+Introduction
+============
+
+Scribe is a server for aggregating log data that's streamed in real
+time from clients. It is designed to be scalable and reliable.
+
+License (See LICENSE file for full license)
+===========================================
+Copyright 2007-2008 Facebook
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+Heirarchy
+=========
+
+scribe/
+
+ aclocal/
+ Contains scripts for building/linking with Boost
+
+ examples/
+ Contains simple examples of using Scribe
+
+ if/
+ Contains Thrift interface for Scribe
+
+ lib/
+ Contains Python package for Scribe
+
+ src/
+ Contains Scribe source
+
+ test/
+ Contain php scripts for testing scribe
+
+
+Requirements
+============
+
+[libevent] Event Notification library
+[boost] Boost C++ library (version 1.36 or later)
+[thrift] Thrift framework
+[fb303] Facebook Bassline (included in thrift/contrib/fb303/)
+ fb303 r697294 or later is required.
+
+These libraries are open source and may be freely obtained, but they are not
+provided as a part of this distribution.
+
+
+Helpful tips:
+-Thrift, fb303, and scribe installation expects python to be installed
+ under /usr. See PY_PREFIX option in 'configure --help' to change this path.
+-Some python installs do not include python site-packages in the default
+ python include path. If python cannot find the installed packages for
+ scribe or fb303, try setting the environment variable PYTHONPATH to the
+ location of the installed packages. This path gets output during
+ 'make install'. (Eg: PYTHONPATH='/usr/lib/python2.5/site-packages').
+
+
+Resources
+=========
+
+More information about Scribe can be obtained on the Scribe webpage at:
+
+ http://developers.facebook.com/scribe
+
+
+To build
+========
+
+./bootstrap.sh <configure options>
+make
+
+(If you have multiple versions of Boost installed, see Boost configure options below.)
+
+Subsequent builds
+=================
+
+./bootstrap <configure options>
+make
+
+OR
+
+./configure <configure options>
+make
+
+NOTE: After the first run with bootstrap.sh you can use "[ ./bootstrap | ./configure ] <options>" followed by "make"
+to create builds with different configurations. "bootstrap" can be passed the same arguments as "configure".
+
+Make sure that if you change configure.ac and|or add macros run "bootstrap.sh".
+to regenerate configure. In short whenever in doubt run "bootstrap.sh".
+
+
+Configure options
+=================
+
+To find all available configure options run
+./configure --help
+
+Use *only* the listed options.
+
+Examples:
+# To disable optimized builds and turn on debug. [ default has been set to optimized]
+./configure --disable-opt
+
+# To disable static libraries and enable shared libraries. [ default has been set to static]
+./configure --disable-static
+
+# To set thrift home to a non-default location
+./configure --with-thriftpath=/myhome/local/thrift
+
+# If Boost is installed in a non-default location or there are multiple Boost versions
+# installed, you will need to specify the Boost path and library names
+./configure --with-boost /usr/local/lib --with-boost-system=boost_system-gcc40-mt-1_36 --with-boost-filesystem=boost_filesystem-gcc40-mt-1_36
+
+
+Install
+=======
+
+as root:
+make install
+
+
+Run
+===
+
+See the examples directory to learn how to use Scribe.
+
+
+Acknowledgements
+================
+The build process for Scribe uses autoconf macros to compile/link with Boost.
+These macros were written by Thomas Porschberg, Michael Tindal, and
+Daniel Casimiro. See ax_boost_base.m4, ax_boost_filesystem.m4, and
+ax_boost_system.m4 in the aclocal subdirectory for more information.
253 acinclude.m4
@@ -0,0 +1,253 @@
+AC_DEFUN([FB_INITIALIZE],
+[
+AM_INIT_AUTOMAKE([ foreign 1.9.5 no-define ])
+if test "x$1" = "xlocalinstall"; then
+wdir=`pwd`
+# To use $wdir undef quote.
+#
+##########
+AC_PREFIX_DEFAULT([`pwd`/install])
+echo
+fi
+AC_PROG_CC
+AC_PROG_CXX
+AC_PROG_RANLIB(RANLIB, ranlib)
+AC_PATH_PROGS(BASH, bash)
+AC_PATH_PROGS(PERL, perl)
+AC_PATH_PROGS(PYTHON, python)
+AC_PATH_PROGS(AR, ar)
+AC_PATH_PROGS(ANT, ant)
+PRODUCT_MK=""
+])
+
+AC_DEFUN([FB_WITH_EXTERNAL_PATH],
+[
+cdir=`pwd`
+AC_MSG_CHECKING([Checking EXTERNAL_PATH set to])
+AC_ARG_WITH([externalpath],
+ [ --with-externalpath=DIR User specified path to external facebook components.],
+ [
+ if test "x${EXTERNAL_PATH}" != "x"; then
+ echo ""
+ echo "ERROR: You have already set EXTERNAL_PATH in your environment"
+ echo "Cannot override it using --with-externalpath. Unset EXTERNAL_PATH to use this option"
+ exit 1
+ fi
+ EXTERNAL_PATH=$withval
+ ],
+ [
+ if test "x${EXTERNAL_PATH}" = "x"; then
+ EXTERNAL_PATH=$1
+ fi
+ ]
+)
+if test "x${EXTERNAL_PATH}" = "x"; then
+ export EXTERNAL_PATH="$cdir/external"
+ GLOBAL_HEADER_MK="include ${EXTERNAL_PATH}/global_header.mk"
+ GLOBAL_FOOTER_MK="include ${EXTERNAL_PATH}/global_footer.mk"
+else
+ export EXTERNAL_PATH
+ GLOBAL_HEADER_MK="include ${EXTERNAL_PATH}/global_header.mk"
+ GLOBAL_FOOTER_MK="include ${EXTERNAL_PATH}/global_footer.mk"
+fi
+AC_MSG_RESULT($EXTERNAL_PATH)
+if test ! -d ${EXTERNAL_PATH}; then
+ echo ""
+ echo "ERROR: EXTERNAL_PATH set to an nonexistent directory ${EXTERNAL_PATH}"
+ exit 1
+fi
+AC_SUBST(EXTERNAL_PATH)
+AC_SUBST(GLOBAL_HEADER_MK)
+AC_SUBST(GLOBAL_FOOTER_MK)
+])
+
+# Set option to enable shared mode. Set DEBUG and OPT for use in Makefile.am.
+AC_DEFUN([FB_ENABLE_DEFAULT_OPT_BUILD],
+[
+AC_MSG_CHECKING([whether to enable optimized build])
+AC_ARG_ENABLE([opt],
+ [ --disable-opt Set up debug mode.],
+ [
+ ENABLED_OPT=$enableval
+ ],
+ [
+ ENABLED_OPT="yes"
+ ]
+)
+if test "$ENABLED_OPT" = "yes"
+then
+ CFLAGS="-Wall -O3"
+ CXXFLAGS="-Wall -O3"
+else
+ CFLAGS="-Wall -g"
+ CXXFLAGS="-Wall -g"
+fi
+AC_MSG_RESULT($ENABLED_OPT)
+AM_CONDITIONAL([OPT], [test "$ENABLED_OPT" = yes])
+AM_CONDITIONAL([DEBUG], [test "$ENABLED_OPT" = no])
+])
+
+# Set option to enable debug mode. Set DEBUG and OPT for use in Makefile.am.
+AC_DEFUN([FB_ENABLE_DEFAULT_DEBUG_BUILD],
+[
+AC_MSG_CHECKING([whether to enable debug build])
+AC_ARG_ENABLE([debug],
+ [ --disable-debug Set up opt mode.],
+ [
+ ENABLED_DEBUG=$enableval
+ ],
+ [
+ ENABLED_DEBUG="yes"
+ ]
+)
+if test "$ENABLED_DEBUG" = "yes"
+then
+ CFLAGS="-Wall -g"
+ CXXFLAGS="-Wall -g"
+else
+ CFLAGS="-Wall -O3"
+ CXXFLAGS="-Wall -O3"
+fi
+AC_MSG_RESULT($ENABLED_DEBUG)
+AM_CONDITIONAL([DEBUG], [test "$ENABLED_DEBUG" = yes])
+AM_CONDITIONAL([OPT], [test "$ENABLED_DEBUG" = no])
+])
+
+# Set option to enable static libs.
+AC_DEFUN([FB_ENABLE_DEFAULT_STATIC],
+[
+SHARED=""
+STATIC=""
+AC_MSG_CHECKING([whether to enable static mode])
+AC_ARG_ENABLE([static],
+ [ --disable-static Set up shared mode.],
+ [
+ ENABLED_STATIC=$enableval
+ ],
+ [
+ ENABLED_STATIC="yes"
+ ]
+)
+if test "$ENABLED_STATIC" = "yes"
+then
+ LTYPE=".a"
+else
+ LTYPE=".so"
+ SHARED_CXXFLAGS="-fPIC"
+ SHARED_CFLAGS="-fPIC"
+ SHARED_LDFLAGS="-shared -fPIC"
+ AC_SUBST(SHARED_CXXFLAGS)
+ AC_SUBST(SHARED_CFLAGS)
+ AC_SUBST(SHARED_LDFLAGS)
+fi
+AC_MSG_RESULT($ENABLED_STATIC)
+AC_SUBST(LTYPE)
+AM_CONDITIONAL([STATIC], [test "$ENABLED_STATIC" = yes])
+AM_CONDITIONAL([SHARED], [test "$ENABLED_STATIC" = no])
+])
+
+# Set option to enable shared libs.
+AC_DEFUN([FB_ENABLE_DEFAULT_SHARED],
+[
+SHARED=""
+STATIC=""
+AC_MSG_CHECKING([whether to enable shared mode])
+AC_ARG_ENABLE([shared],
+ [ --disable-shared Set up static mode.],
+ [
+ ENABLED_SHARED=$enableval
+ ],
+ [
+ ENABLED_SHARED="yes"
+ ]
+)
+if test "$ENABLED_SHARED" = "yes"
+then
+ LTYPE=".so"
+ SHARED_CXXFLAGS="-fPIC"
+ SHARED_CFLAGS="-fPIC"
+ SHARED_LDFLAGS="-shared -fPIC"
+ AC_SUBST(SHARED_CXXFLAGS)
+ AC_SUBST(SHARED_CFLAGS)
+ AC_SUBST(SHARED_LDFLAGS)
+else
+ LTYPE=".a"
+fi
+AC_MSG_RESULT($ENABLED_SHARED)
+AC_SUBST(LTYPE)
+AM_CONDITIONAL([SHARED], [test "$ENABLED_SHARED" = yes])
+AM_CONDITIONAL([STATIC], [test "$ENABLED_SHARED" = no])
+])
+
+# Generates define flags and conditionals as specified by user.
+# This gets enabled *only* if user selects --enable-<FEATURE> otion.
+AC_DEFUN([FB_ENABLE_FEATURE],
+[
+ENABLE=""
+flag="$1"
+value="$3"
+AC_MSG_CHECKING([whether to enable $1])
+AC_ARG_ENABLE([$2],
+ [ --enable-$2 Enable $2.],
+ [
+ ENABLE=$enableval
+ ],
+ [
+ ENABLE="no"
+ ]
+)
+AM_CONDITIONAL([$1], [test "$ENABLE" = yes])
+if test "$ENABLE" = "yes"
+then
+ if test "x${value}" = "x"
+ then
+ AC_DEFINE([$1])
+ else
+ AC_DEFINE_UNQUOTED([$1], [$value])
+ fi
+fi
+AC_MSG_RESULT($ENABLE)
+])
+
+
+# can also use eval $2=$withval;AC_SUBST($2)
+AC_DEFUN([FB_WITH_PATH],
+[
+USRFLAG=""
+USRFLAG=$1
+AC_MSG_CHECKING([Checking $1 set to])
+AC_ARG_WITH([$2],
+ [ --with-$2=DIR User specified path.],
+ [
+ LOC=$withval
+ eval $USRFLAG=$withval
+ ],
+ [
+ LOC=$3
+ eval $USRFLAG=$3
+ ]
+)
+AC_SUBST([$1])
+AC_MSG_RESULT($LOC)
+])
+
+AC_DEFUN([FB_SET_FLAG_VALUE],
+[
+SETFLAG=""
+AC_MSG_CHECKING([Checking $1 set to])
+SETFLAG=$1
+eval $SETFLAG=\"$2\"
+AC_SUBST([$SETFLAG])
+AC_MSG_RESULT($2)
+])
+
+# NOTES
+# if using if else bourne stmt you must have more than a macro in it.
+# EX1 is not correct. EX2 is correct
+# EX1: if test "$XX" = "yes"; then
+# AC_SUBST(xx)
+# fi
+# EX2: if test "$XX" = "yes"; then
+# xx="foo"
+# AC_SUBST(xx)
+# fi
198 aclocal/ax_boost_base.m4
@@ -0,0 +1,198 @@
+##### http://autoconf-archive.cryp.to/ax_boost_base.html
+#
+# SYNOPSIS
+#
+# AX_BOOST_BASE([MINIMUM-VERSION])
+#
+# DESCRIPTION
+#
+# Test for the Boost C++ libraries of a particular version (or newer)
+#
+# If no path to the installed boost library is given the macro
+# searchs under /usr, /usr/local, /opt and /opt/local and evaluates
+# the $BOOST_ROOT environment variable. Further documentation is
+# available at <http://randspringer.de/boost/index.html>.
+#
+# This macro calls:
+#
+# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS)
+#
+# And sets:
+#
+# HAVE_BOOST
+#
+# LAST MODIFICATION
+#
+# 2007-07-28
+#
+# COPYLEFT
+#
+# Copyright (c) 2007 Thomas Porschberg <thomas@randspringer.de>
+#
+# Copying and distribution of this file, with or without
+# modification, are permitted in any medium without royalty provided
+# the copyright notice and this notice are preserved.
+
+AC_DEFUN([AX_BOOST_BASE],
+[
+AC_ARG_WITH([boost],
+ AS_HELP_STRING([--with-boost@<:@=DIR@:>@], [use boost (default is yes) - it is possible to specify the root directory for boost (optional)]),
+ [
+ if test "$withval" = "no"; then
+ want_boost="no"
+ elif test "$withval" = "yes"; then
+ want_boost="yes"
+ ac_boost_path=""
+ else
+ want_boost="yes"
+ ac_boost_path="$withval"
+ fi
+ ],
+ [want_boost="yes"])
+
+if test "x$want_boost" = "xyes"; then
+ boost_lib_version_req=ifelse([$1], ,1.20.0,$1)
+ boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'`
+ boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'`
+ boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'`
+ boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'`
+ if test "x$boost_lib_version_req_sub_minor" = "x" ; then
+ boost_lib_version_req_sub_minor="0"
+ fi
+ WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor`
+ AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req)
+ succeeded=no
+
+ dnl first we check the system location for boost libraries
+ dnl this location ist chosen if boost libraries are installed with the --layout=system option
+ dnl or if you install boost with RPM
+ if test "$ac_boost_path" != ""; then
+ BOOST_LDFLAGS="-L$ac_boost_path/lib"
+ BOOST_CPPFLAGS="-I$ac_boost_path/include"
+ else
+ for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do
+ if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then
+ BOOST_LDFLAGS="-L$ac_boost_path_tmp/lib"
+ BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include"
+ break;
+ fi
+ done
+ fi
+
+ CPPFLAGS_SAVED="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+
+ LDFLAGS_SAVED="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_LANG_PUSH(C++)
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+ @%:@include <boost/version.hpp>
+ ]], [[
+ #if BOOST_VERSION >= $WANT_BOOST_VERSION
+ // Everything is okay
+ #else
+ # error Boost version is too old
+ #endif
+ ]])],[
+ AC_MSG_RESULT(yes)
+ succeeded=yes
+ found_system=yes
+ ],[
+ ])
+ AC_LANG_POP([C++])
+
+
+
+ dnl if we found no boost with system layout we search for boost libraries
+ dnl built and installed without the --layout=system option or for a staged(not installed) version
+ if test "x$succeeded" != "xyes"; then
+ _version=0
+ if test "$ac_boost_path" != ""; then
+ BOOST_LDFLAGS="-L$ac_boost_path/lib"
+ if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
+ for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
+ _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
+ V_CHECK=`expr $_version_tmp \> $_version`
+ if test "$V_CHECK" = "1" ; then
+ _version=$_version_tmp
+ fi
+ VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
+ BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE"
+ done
+ fi
+ else
+ for ac_boost_path in /usr /usr/local /opt /opt/local ; do
+ if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
+ for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
+ _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
+ V_CHECK=`expr $_version_tmp \> $_version`
+ if test "$V_CHECK" = "1" ; then
+ _version=$_version_tmp
+ best_path=$ac_boost_path
+ fi
+ done
+ fi
+ done
+
+ VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
+ BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE"
+ BOOST_LDFLAGS="-L$best_path/lib"
+
+ if test "x$BOOST_ROOT" != "x"; then
+ if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/lib" && test -r "$BOOST_ROOT/stage/lib"; then
+ version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'`
+ stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'`
+ stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'`
+ V_CHECK=`expr $stage_version_shorten \>\= $_version`
+ if test "$V_CHECK" = "1" ; then
+ AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT)
+ BOOST_CPPFLAGS="-I$BOOST_ROOT"
+ BOOST_LDFLAGS="-L$BOOST_ROOT/stage/lib"
+ fi
+ fi
+ fi
+ fi
+
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_LANG_PUSH(C++)
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+ @%:@include <boost/version.hpp>
+ ]], [[
+ #if BOOST_VERSION >= $WANT_BOOST_VERSION
+ // Everything is okay
+ #else
+ # error Boost version is too old
+ #endif
+ ]])],[
+ AC_MSG_RESULT(yes)
+ succeeded=yes
+ found_system=yes
+ ],[
+ ])
+ AC_LANG_POP([C++])
+ fi
+
+ if test "$succeeded" != "yes" ; then
+ if test "$_version" = "0" ; then
+ AC_MSG_ERROR([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in <boost/version.hpp>. See http://randspringer.de/boost for more documentation.]])
+ else
+ AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).])
+ fi
+ else
+ AC_SUBST(BOOST_CPPFLAGS)
+ AC_SUBST(BOOST_LDFLAGS)
+ AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available])
+ fi
+
+ CPPFLAGS="$CPPFLAGS_SAVED"
+ LDFLAGS="$LDFLAGS_SAVED"
+fi
+
+])
110 aclocal/ax_boost_filesystem.m4
@@ -0,0 +1,110 @@
+# ===========================================================================
+# http://autoconf-archive.cryp.to/ax_boost_filesystem.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_BOOST_FILESYSTEM
+#
+# DESCRIPTION
+#
+# Test for Filesystem library from the Boost C++ libraries. The macro
+# requires a preceding call to AX_BOOST_BASE. Further documentation is
+# available at <http://randspringer.de/boost/index.html>.
+#
+# This macro calls:
+#
+# AC_SUBST(BOOST_FILESYSTEM_LIB)
+#
+# And sets:
+#
+# HAVE_BOOST_FILESYSTEM
+#
+# LAST MODIFICATION
+#
+# 2008-04-12
+#
+# COPYLEFT
+#
+# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
+# Copyright (c) 2008 Michael Tindal
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved.
+
+AC_DEFUN([AX_BOOST_FILESYSTEM],
+[
+ AC_ARG_WITH([boost-filesystem],
+ AS_HELP_STRING([--with-boost-filesystem@<:@=special-lib@:>@],
+ [use the Filesystem library from boost - it is possible to specify a certain library for the linker
+ e.g. --with-boost-filesystem=boost_filesystem-gcc-mt ]),
+ [
+ if test "$withval" = "no"; then
+ want_boost="no"
+ elif test "$withval" = "yes"; then
+ want_boost="yes"
+ ax_boost_user_filesystem_lib=""
+ else
+ want_boost="yes"
+ ax_boost_user_filesystem_lib="$withval"
+ fi
+ ],
+ [want_boost="yes"]
+ )
+
+ if test "x$want_boost" = "xyes"; then
+ AC_REQUIRE([AC_PROG_CC])
+ CPPFLAGS_SAVED="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+
+ LDFLAGS_SAVED="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_CACHE_CHECK(whether the Boost::Filesystem library is available,
+ ax_cv_boost_filesystem,
+ [AC_LANG_PUSH([C++])
+ AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/filesystem/path.hpp>]],
+ [[using namespace boost::filesystem;
+ path my_path( "foo/bar/data.txt" );
+ return 0;]]),
+ ax_cv_boost_filesystem=yes, ax_cv_boost_filesystem=no)
+ AC_LANG_POP([C++])
+ ])
+ if test "x$ax_cv_boost_filesystem" = "xyes"; then
+ AC_DEFINE(HAVE_BOOST_FILESYSTEM,,[define if the Boost::Filesystem library is available])
+ BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+ if test "x$ax_boost_user_filesystem_lib" = "x"; then
+ for libextension in `ls $BOOSTLIBDIR/libboost_filesystem*.{so,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_filesystem.*\)\.so.*$;\1;' -e 's;^lib\(boost_filesystem.*\)\.a*$;\1;'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break],
+ [link_filesystem="no"])
+ done
+ if test "x$link_program_options" != "xyes"; then
+ for libextension in `ls $BOOSTLIBDIR/boost_filesystem*.{dll,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_filesystem.*\)\.dll.*$;\1;' -e 's;^\(boost_filesystem.*\)\.a*$;\1;'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break],
+ [link_filesystem="no"])
+ done
+ fi
+ else
+ for ax_lib in $ax_boost_user_filesystem_lib boost_filesystem-$ax_boost_user_filesystem_lib; do
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break],
+ [link_filesystem="no"])
+ done
+
+ fi
+ if test "x$link_filesystem" != "xyes"; then
+ AC_MSG_ERROR(Could not link against $ax_lib !)
+ fi
+ fi
+
+ CPPFLAGS="$CPPFLAGS_SAVED"
+ LDFLAGS="$LDFLAGS_SAVED"
+ fi
+])
118 aclocal/ax_boost_system.m4
@@ -0,0 +1,118 @@
+# ===========================================================================
+# http://autoconf-archive.cryp.to/ax_boost_system.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_BOOST_SYSTEM
+#
+# DESCRIPTION
+#
+# Test for System library from the Boost C++ libraries. The macro requires
+# a preceding call to AX_BOOST_BASE. Further documentation is available at
+# <http://randspringer.de/boost/index.html>.
+#
+# This macro calls:
+#
+# AC_SUBST(BOOST_SYSTEM_LIB)
+#
+# And sets:
+#
+# HAVE_BOOST_SYSTEM
+#
+# LAST MODIFICATION
+#
+# 2008-04-12
+#
+# COPYLEFT
+#
+# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
+# Copyright (c) 2008 Michael Tindal
+# Copyright (c) 2008 Daniel Casimiro <dan.casimiro@gmail.com>
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved.
+
+AC_DEFUN([AX_BOOST_SYSTEM],
+[
+ AC_ARG_WITH([boost-system],
+ AS_HELP_STRING([--with-boost-system@<:@=special-lib@:>@],
+ [use the System library from boost - it is possible to specify a certain library for the linker
+ e.g. --with-boost-system=boost_system-gcc-mt ]),
+ [
+ if test "$withval" = "no"; then
+ want_boost="no"
+ elif test "$withval" = "yes"; then
+ want_boost="yes"
+ ax_boost_user_system_lib=""
+ else
+ want_boost="yes"
+ ax_boost_user_system_lib="$withval"
+ fi
+ ],
+ [want_boost="yes"]
+ )
+
+ if test "x$want_boost" = "xyes"; then
+ AC_REQUIRE([AC_PROG_CC])
+ AC_REQUIRE([AC_CANONICAL_BUILD])
+ CPPFLAGS_SAVED="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+
+ LDFLAGS_SAVED="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_CACHE_CHECK(whether the Boost::System library is available,
+ ax_cv_boost_system,
+ [AC_LANG_PUSH([C++])
+ CXXFLAGS_SAVE=$CXXFLAGS
+
+ AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/system/error_code.hpp>]],
+ [[boost::system::system_category]]),
+ ax_cv_boost_system=yes, ax_cv_boost_system=no)
+ CXXFLAGS=$CXXFLAGS_SAVE
+ AC_LANG_POP([C++])
+ ])
+ if test "x$ax_cv_boost_system" = "xyes"; then
+ AC_SUBST(BOOST_CPPFLAGS)
+
+ AC_DEFINE(HAVE_BOOST_SYSTEM,,[define if the Boost::System library is available])
+ BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+
+ LDFLAGS_SAVE=$LDFLAGS
+ if test "x$ax_boost_user_system_lib" = "x"; then
+ for libextension in `ls $BOOSTLIBDIR/libboost_system*.{so,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_system.*\)\.so.*$;\1;' -e 's;^lib\(boost_system.*\)\.a*$;\1;'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break],
+ [link_system="no"])
+ done
+ if test "x$link_system" != "xyes"; then
+ for libextension in `ls $BOOSTLIBDIR/boost_system*.{dll,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_system.*\)\.dll.*$;\1;' -e 's;^\(boost_system.*\)\.a*$;\1;'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break],
+ [link_system="no"])
+ done
+ fi
+
+ else
+ for ax_lib in $ax_boost_user_system_lib boost_system-$ax_boost_user_system_lib; do
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break],
+ [link_system="no"])
+ done
+
+ fi
+ if test "x$link_system" = "xno"; then
+ AC_MSG_ERROR(Could not link against $ax_lib !)
+ fi
+ fi
+
+ CPPFLAGS="$CPPFLAGS_SAVED"
+ LDFLAGS="$LDFLAGS_SAVED"
+ fi
+])
7 bootstrap.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# To be safe include -I flag
+aclocal -I ./aclocal
+automake -a
+autoconf
+./configure --config-cache $*
102 configure.ac
@@ -0,0 +1,102 @@
+# Autoconf input file
+# $Id$
+
+# AC - autoconf
+# FB - facebook
+
+#########################################################################
+# DO NOT TOUCH EXCEPT TO CHANGE REV# IN AC_INIT
+
+AC_PREREQ(2.52)
+AC_INIT([scribe], [1.5.0])
+#AC_CONFIG_AUX_DIR([/usr/share/automake-1.9])
+# To install locally
+FB_INITIALIZE([localinstall])
+AC_PREFIX_DEFAULT([/usr/local])
+
+############################################################################
+# User Configurable. Change With CAUTION!
+# User can include custom makefile rules. Uncomment and update only <name> in PRODUCT_MK.
+# Include where appropriate in any Makefile.am as @PRODUCT_MK@
+
+#PRODUCT_MK="include ${EXTERNAL_PATH}/shared/build/<name>.mk"
+
+# Default path to external Facebook components and shared build toools I.e fb303 etc.
+# To point to other locations set environment variable EXTERNAL_PATH.
+# To change the current default you must change bootstrap.sh.
+FB_WITH_EXTERNAL_PATH([`pwd`])
+
+AC_ARG_VAR([PY_PREFIX], [Prefix for installing Python modules.
+ (Normal --prefix is ignored for Python because
+ Python has different conventions.)
+ Default = "/usr"])
+AS_IF([test "x$PY_PREFIX" = x], [PY_PREFIX="/usr"])
+
+##########################################################################
+# User Configurable
+
+# Pre-defined macro to set opt build mode. Run with --disable-shared option to turn off optimization.
+FB_ENABLE_DEFAULT_OPT_BUILD
+
+# Predefined macro to set static library mode. Run with --disable-static option to turn off static lib mode.
+FB_ENABLE_DEFAULT_STATIC
+
+# Personalized feature generator. Creates defines/conditionals and --enable --disable command line options.
+# FB_ENABLE_FEATURE([FEATURE], [feature]) OR FB_ENABLE_FEATURE([FEATURE], [feature], [\"<value>\"])
+
+# Example: Macro supplies -DFACEBOOK at compile time and "if FACEBOOK endif" capabilities.
+FB_ENABLE_FEATURE([FACEBOOK], [facebook])
+
+# Personalized path generator Sets default paths. Provides --with-xx=DIR options.
+# FB_WITH_PATH([<var>_home], [<var>path], [<default location>]
+
+# Example: sets $(thrift_home) variable with default path set to /usr/local.
+FB_WITH_PATH([thrift_home], [thriftpath], [/usr/local])
+FB_WITH_PATH([jvm_lib], [jvmpath], [/usr/local/java/jre/lib/amd64/server])
+FB_WITH_PATH([fb303_home], [fb303path], [/usr/local])
+FB_WITH_PATH([smc_home], [smcpath], [${EXTERNAL_PATH}/services/trunk/src])
+FB_WITH_PATH([fb_home], [fbpath], [${EXTERNAL_PATH}/libfacebook])
+
+# Require boost 1.36 with system and filesytem libraries
+AX_BOOST_BASE([1.36])
+AX_BOOST_SYSTEM
+AX_BOOST_FILESYSTEM
+
+# Generates Makefile from Makefile.am. Modify when new subdirs are added.
+# Change Makefile.am also to add subdirectly.
+AC_CONFIG_FILES(Makefile src/Makefile lib/py/Makefile)
+
+############################################################################
+# DO NOT TOUCH.
+
+AC_SUBST(PRODUCT_MK)
+AC_OUTPUT
+
+#############################################################################
+######### FINISH ############################################################
+
+echo "EXTERNAL_PATH $EXTERNAL_PATH"
+
+#make clean
+
+#
+# NOTES FOR USER
+# Short cut to create conditional flags.
+#enable_facebook="yes"
+#AM_CONDITIONAL([FACEBOOK], [test "$enable_facebook" = yes])
+
+# Enable options with --enable and --disable configurable.
+#AC_MSG_CHECKING([whether to enable FACEBOOK])
+#FACEBOOK=""
+#AC_ARG_ENABLE([facebook],
+# [ --enable-facebook Enable facebook.],
+# [
+# ENABLE_FACEBOOK=$enableval
+# ],
+# [
+# ENABLE_FACEBOOK="no"
+# ]
+#)
+#AM_CONDITIONAL([FACEBOOK], [test "$ENABLE_FACEBOOK" = yes])
+#AC_MSG_RESULT($ENABLE_FACEBOOK)
+
172 examples/README
@@ -0,0 +1,172 @@
+## Copyright (c) 2007-2008 Facebook
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## See accompanying file LICENSE or visit the Scribe site at:
+## http://developers.facebook.com/scribe/
+
+
+# This file contains a couple of simple examples of how to configure and use
+# Scribe.
+#
+# Example code in this directory:
+# scribe_cat: a simple example of a client that can send messages to Scribe
+# scribe_ctrl: a script that manages a running Scribe instance (requires root)
+# example1.conf: sample configuration file for running Example 1
+# example2.conf: sample configuration file for running Example 2
+
+#
+# EXAMPLE 1
+#
+
+# This is a simple example that shows how to configure and send messages to
+# Scribe.
+
+#Create a directory to log messages:
+mkdir /tmp/scribetest
+
+#Start scribe using the configuration in example1.conf:
+src/scribed examples/example1.conf
+
+#From another terminal, use scribe_cat to send a message to scribe:
+echo "hello world" | ./scribe_cat test
+
+#If the previous command failed, make sure you did a 'make install' from the
+#root scribe directory and that $PYTHONPATH is set correctly(see README.BUILD)
+
+#Verify that the message got logged:
+cat /tmp/scribetest/test/test_current
+
+#Check the status of scribe (requires root):
+./scribe_ctrl status
+
+#Check scribe's counters (you should see 1 message 'received good'):
+./scribe_ctrl counters
+
+#Shutdown scribe:
+./scribe_ctrl stop
+
+
+#
+# Example 2
+#
+
+# This example shows you how to log messages between multiple Scribe instances.
+# In this example, we will run each Scribe server on a different port to simulate
+# running Scribe on multiple machines.
+
+ 'client' 'central'
+---------------------------- --------------------
+| Port 1464 | | Port 1463 |
+| ---------------- | | ---------------- |
+| -> | scribe server |--|--->| | scribe server | |
+| ---------------- | | ---------------- |
+| | | | | | |
+| temp file | | | temp file |
+|--------------------------- |-------------------
+ |
+ -------------------
+ | /tmp/scribetest/ |
+ -------------------
+
+
+#Create a directory for the second scribe instance:
+mkdir /tmp/scribetest2
+
+#Start up the 'central' instance of Scribe on port 1463 to write messages to disk
+#(See example2central.conf):
+src/scribed examples/example2central.conf
+
+#Start up the 'client' instance of Scribe on port 1464 to forward messages to
+#the 'central' Scribe server (See example2client.conf):
+src/scribed examples/example2client.conf
+
+#Use scribe_cat to send some messages to the 'client' Scribe instance:
+echo "test message" | ./scribe_cat -h localhost:1464 test2
+
+echo "this message will be ignored" | ./scribe_cat -h localhost:1464 ignore_me
+
+echo "123:this message will be bucketed" | ./scribe_cat -h localhost:1464 bucket_me
+
+#The first message will be logged similar to example 1.
+#The second message will not get logged.
+#The third message will be bucketized into 1 of 5 buckets
+#(See example2central.conf)
+
+#Verify that the first message got logged:
+cat /tmp/scribetest/test2/test2_current
+
+#Verify that the third message got logged into a subdirectory:
+cat /tmp/scribetest/bucket*/bucket_me_current
+
+#Check the status and counters of both instances:
+./scribe_ctrl status 1463
+./scribe_ctrl status 1464
+./scribe_ctrl counters 1463
+./scribe_ctrl counters 1464
+
+#Shutdown both servers:
+./scribe_ctrl stop 1463
+./scribe_ctrl stop 1464
+
+
+#
+# Example 3
+#
+
+# Test Scribe buffering
+
+#Startup the two Scribe instances used in Example 2.
+#Start the 'central' server first:
+src/scribed examples/example2central.conf
+
+#Then start the 'client':
+src/scribed examples/example2client.conf
+
+#Log a message to the 'client' Scribe instance:
+echo "test message 1" | ./scribe_cat -h localhost:1464 test3
+
+#Verify that the message got logged:
+cat /tmp/scribetest/test3/test3_current
+
+#Stop the 'central' Scribe instance:
+./scribe_ctrl stop 1463
+
+#Attempting to check the status of this server will return failure since it not running:
+./scribe_ctrl status 1463
+
+#Try to Log another message:
+echo "test message 2" | ./scribe_cat -h localhost:1464 test3
+
+#This message will be buffered by the 'client' since it cannot be forwarded to
+#the 'central' server. Scribe will keep retrying until it is able to send.
+
+#After a couple seconds, the status of the 'client' will be set to a warning message:
+./scribe_ctrl status 1464
+
+#Try to Log yet another message(which will also get buffered):
+echo "test message 3" | ./scribe_cat -h localhost:1464 test3
+
+#Restart the 'central' instance:
+src/scribed examples/example2central.conf
+
+#Wait for both Scribe instance's statuses to change to ALIVE:
+./scribe_ctrl status 1463
+./scribe_ctrl status 1464
+
+#Verify that all 3 messages have now been received by the 'central' server:
+cat /tmp/scribetest/test3/test3_current
+
+#Shutdown:
+./scribe_ctrl stop 1463
+./scribe_ctrl stop 1464
57 examples/example1.conf
@@ -0,0 +1,57 @@
+## Copyright (c) 2007-2008 Facebook
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## See accompanying file LICENSE or visit the Scribe site at:
+## http://developers.facebook.com/scribe/
+
+
+##
+## Sample Scribe configuration
+##
+
+# This file configures Scribe to listen for messages on port 1463 and write
+# them to /tmp/scribetest
+
+port=1463
+max_msg_per_second=2000000
+check_interval=3
+
+# DEFAULT
+<store>
+category=default
+type=buffer
+
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+type=file
+fs_type=std
+file_path=/tmp/scribetest
+base_filename=thisisoverwritten
+max_size=1000000
+add_newlines=1
+</primary>
+
+<secondary>
+type=file
+fs_type=std
+file_path=/tmp
+base_filename=thisisoverwritten
+max_size=3000000
+</secondary>
+</store>
108 examples/example2central.conf
@@ -0,0 +1,108 @@
+## Copyright (c) 2007-2008 Facebook
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## See accompanying file LICENSE or visit the Scribe site at:
+## http://developers.facebook.com/scribe/
+
+
+##
+## Sample Scribe configuration
+##
+
+# This file configures Scribe to listen for messages on port 1463 and write
+# them to /tmp/scribetest
+#
+# This configuration also tells Scribe to discard messages with a category
+# that begins with 'ignore'.
+#
+# If the message category is 'bucket_me', Scribe will hash this message to
+# 1 of 5 buckets.
+
+port=1463
+max_msg_per_second=2000000
+check_interval=3
+
+# IGNORE* - discards messages for categories that begin with 'ignore'
+<store>
+category=ignore*
+type=null
+</store>
+
+
+# BUCKET_ME - write 'bucket_me' messages to 1 of 5 subdirectories
+<store>
+category=bucket_me
+type=buffer
+
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+type=bucket
+num_buckets=5
+bucket_subdir=bucket
+bucket_type=key_hash
+delimiter=58
+# This will hash based on the part of the message before the first ':' (char(58))
+
+<bucket>
+type=file
+fs_type=std
+file_path=/tmp/scribetest
+base_filename=bucket_me
+max_size=10000
+</bucket>
+</primary>
+
+<secondary>
+type=file
+fs_type=std
+file_path=/tmp
+base_filename=bucket_me
+max_size=30000
+</secondary>
+</store>
+
+
+# DEFAULT - write all other categories to /tmp/scribetest
+<store>
+category=default
+type=buffer
+
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+type=file
+fs_type=std
+file_path=/tmp/scribetest
+base_filename=thisisoverwritten
+max_size=1000000
+</primary>
+
+<secondary>
+type=file
+fs_type=std
+file_path=/tmp
+base_filename=thisisoverwritten
+max_size=3000000
+</secondary>
+</store>
+
59 examples/example2client.conf
@@ -0,0 +1,59 @@
+## Copyright (c) 2007-2008 Facebook
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## See accompanying file LICENSE or visit the Scribe site at:
+## http://developers.facebook.com/scribe/
+
+
+##
+## Sample Scribe configuration
+##
+
+# This file configures Scribe to listen for messages on port 1464 and
+# attempt to forward all messages to another Scribe instance on port 1463.
+# If Scribe is unable to forward the messages to port 1463, it will buffer
+# them on disk and keep retrying.
+
+
+port=1464
+max_msg_per_second=2000000
+check_interval=3
+
+
+# DEFAULT - forward all messages to Scribe on port 1463
+<store>
+category=default
+type=buffer
+
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=1
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+type=network
+remote_host=localhost
+remote_port=1463
+</primary>
+
+<secondary>
+type=file
+fs_type=std
+file_path=/tmp/scribetest2
+base_filename=thisisoverwritten
+max_size=3000000
+</secondary>
+</store>
+
60 examples/scribe_cat
@@ -0,0 +1,60 @@
+#!/usr/bin/python
+
+## Copyright (c) 2007-2008 Facebook
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## See accompanying file LICENSE or visit the Scribe site at:
+## http://developers.facebook.com/scribe/
+
+
+'''scribe_cat: A simple script for sending messages to scribe.'''
+
+import sys
+from scribe import scribe
+from thrift.transport import TTransport, TSocket
+from thrift.protocol import TBinaryProtocol
+
+if len(sys.argv) == 2:
+ category = sys.argv[1]
+ host = '127.0.0.1'
+ port = 1463
+elif len(sys.argv) == 4 and sys.argv[1] == '-h':
+ category = sys.argv[3]
+ host_port = sys.argv[2].split(':')
+ host = host_port[0]
+ if len(host_port) > 1:
+ port = int(host_port[1])
+ else:
+ port = 1463
+else:
+ sys.exit('usage (message is stdin): scribe_cat [-h host[:port]] category')
+
+log_entry = scribe.LogEntry(dict(category=category, message=sys.stdin.read()))
+
+socket = TSocket.TSocket(host=host, port=port)
+transport = TTransport.TFramedTransport(socket)
+protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False)
+client = scribe.Client(iprot=protocol, oprot=protocol)
+
+transport.open()
+result = client.Log(messages=[log_entry])
+transport.close()
+
+if result == scribe.ResultCode.OK:
+ sys.exit()
+elif result == scribe.ResultCode.TRY_LATER:
+ print >> sys.stderr, "TRY_LATER"
+ sys.exit(84) # 'T'
+else:
+ sys.exit("Unknown error code.")
46 examples/scribe_ctrl
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+## Copyright (c) 2007-2008 Facebook
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## See accompanying file LICENSE or visit the Scribe site at:
+## http://developers.facebook.com/scribe/
+
+'''scribe_ctrl: A simple script for running and monitoring scribe.'''
+
+import sys
+from fb303_scripts import *
+
+# thrift python packages need to be installed
+import thrift
+from thrift import protocol, transport
+from thrift.transport import TTransport
+from thrift.protocol import TBinaryProtocol
+
+if (len(sys.argv) > 2):
+ port = int(sys.argv[2])
+else:
+ port = 1463
+
+if (len(sys.argv) > 1):
+ retval = fb303_simple_mgmt.service_ctrl(sys.argv[1],
+ port,
+ trans_factory = TTransport.TFramedTransportFactory(),
+ prot_factory = TBinaryProtocol.TBinaryProtocolFactory())
+ sys.exit(retval)
+
+else:
+ print 'Usage: scribe_ctrl command [port]'
+ print ' commands: stop counters status version name alive'
+ sys.exit(2)
2 global_footer.mk
@@ -0,0 +1,2 @@
+thriftstyle : $(XBUILT_SOURCES)
+
19 global_header.mk
@@ -0,0 +1,19 @@
+#define thrift_template
+# $(1) : $(2)
+# $$(THRIFT) $(3) $(4) $(5) $(6) $(7) $(8) $$<
+#endef
+
+define thrift_template
+XTARGET := $(shell perl -e '@val = split("\/","$(2)"); $$last = pop(@val);split("\\.",$$last);print "$(1)/"."gen-cpp/"."@_[0]"."_types.cpp\n"' )
+
+ifneq ($$(XBUILT_SOURCES),)
+ XBUILT_SOURCES := $$(XBUILT_SOURCES) $$(XTARGET)
+else
+ XBUILT_SOURCES := $$(XTARGET)
+endif
+$$(XTARGET) : $(2)
+ $$(THRIFT) -o $1 $3 $$<
+endef
+
+clean-common:
+ rm -rf gen-*
39 if/scribe.thrift
@@ -0,0 +1,39 @@
+#!/usr/local/bin/thrift --cpp --php
+
+## Copyright (c) 2007-2008 Facebook
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## See accompanying file LICENSE or visit the Scribe site at:
+## http://developers.facebook.com/scribe/
+
+include "fb303/if/fb303.thrift"
+
+namespace cpp scribe.thrift
+
+enum ResultCode
+{
+ OK,
+ TRY_LATER
+}
+
+struct LogEntry
+{
+ 1: string category,
+ 2: string message
+}
+
+service scribe extends fb303.FacebookService
+{
+ ResultCode Log(1: list<LogEntry> messages);
+}
22 lib/py/Makefile.am
@@ -0,0 +1,22 @@
+DESTDIR ?= /
+EXTRA_DIST = setup.py scribe
+
+all:
+
+all-local:
+ $(PYTHON) setup.py build
+
+# We're ignoring prefix here because site-packages seems to be
+# the equivalent of /usr/local/lib in Python land.
+# Old version (can't put inline because it's not portable).
+#$(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR) $(PYTHON_SETUPUTIL_ARGS)
+install-exec-hook:
+ $(PYTHON) setup.py install --root=$(DESTDIR) --prefix=$(PY_PREFIX) $(PYTHON_SETUPUTIL_ARGS)
+
+
+
+clean: clean-local
+
+clean-local:
+
+check-local: all
1 lib/py/scribe/__init__.py
@@ -0,0 +1 @@
+__all__ = ['ttypes', 'constants', 'scribe']
9 lib/py/scribe/constants.py
@@ -0,0 +1,9 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+from thrift.protocol.TProtocol import *
+from ttypes import *
+
60 lib/py/scribe/scribe-remote
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+import sys
+import pprint
+from thrift.transport import TTransport
+from thrift.transport import TSocket
+from thrift.protocol import TBinaryProtocol
+
+import scribe
+from ttypes import *
+
+if len(sys.argv) <= 1 or sys.argv[1] == '--help':
+ print ''
+ print 'Usage: ' + sys.argv[0] + ' [-h host:port] [-f[ramed]] function [arg1 [arg2...]]'
+ print ''
+ print 'Functions:'
+ print ' ResultCode Log( messages)'
+ print ''
+ sys.exit(0)
+
+pp = pprint.PrettyPrinter(indent = 2)
+host = 'localhost'
+port = 9090
+framed = False
+argi = 1
+
+if sys.argv[1] == '-h':
+ parts = sys.argv[2].split(':')
+ host = parts[0]
+ port = int(parts[1])
+ argi = 3
+
+if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed':
+ framed = True
+ argi += 1
+
+cmd = sys.argv[argi]
+args = sys.argv[argi+1:]
+
+socket = TSocket.TSocket(host, port)
+if framed:
+ transport = TTransport.TFramedTransport(socket)
+else:
+ transport = TTransport.TBufferedTransport(socket)
+protocol = TBinaryProtocol.TBinaryProtocol(transport)
+client = scribe.Client(protocol)
+transport.open()
+
+if cmd == 'Log':
+ if len(args) != 1:
+ print 'Log requires 1 args'
+ sys.exit(1)
+ pp.pprint(client.Log(eval(args[0]),))
+
+transport.close()
157 lib/py/scribe/scribe.py
@@ -0,0 +1,157 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+from thrift.protocol.TProtocol import *
+import fb303.FacebookService
+from ttypes import *
+from thrift.Thrift import TProcessor
+
+class Iface(fb303.FacebookService.Iface):
+ def Log(self, messages):
+ pass
+
+
+class Client(fb303.FacebookService.Client, Iface):
+ def __init__(self, iprot, oprot=None):
+ fb303.FacebookService.Client.__init__(self, iprot, oprot)
+
+ def Log(self, messages):
+ self.send_Log(messages)
+ return self.recv_Log()
+
+ def send_Log(self, messages):
+ self._oprot.writeMessageBegin('Log', TMessageType.CALL, self._seqid)
+ args = Log_args()
+ args.messages = messages
+ args.write(self._oprot)
+ self._oprot.writeMessageEnd()
+ self._oprot.trans.flush()
+
+ def recv_Log(self, ):
+ (fname, mtype, rseqid) = self._iprot.readMessageBegin()
+ result = Log_result()
+ result.read(self._iprot)
+ self._iprot.readMessageEnd()
+ if result.success != None:
+ return result.success
+ raise Exception("Log failed: unknown result");
+
+
+class Processor(fb303.FacebookService.Processor, Iface, TProcessor):
+ def __init__(self, handler):
+ fb303.FacebookService.Processor.__init__(self, handler)
+ self._processMap["Log"] = Processor.process_Log
+
+ def process(self, iprot, oprot):
+ (name, type, seqid) = iprot.readMessageBegin()
+ if name not in self._processMap:
+ print 'Unknown function %s' % (name)
+ else:
+ self._processMap[name](self, seqid, iprot, oprot)
+ return True
+
+ def process_Log(self, seqid, iprot, oprot):
+ args = Log_args()
+ args.read(iprot)
+ iprot.readMessageEnd()
+ result = Log_result()
+ result.success = self._handler.Log(args.messages)
+ oprot.writeMessageBegin("Log", TMessageType.REPLY, seqid)
+ result.write(oprot)
+ oprot.writeMessageEnd()
+ oprot.trans.flush()
+
+
+# HELPER FUNCTIONS AND STRUCTURES
+
+class Log_args:
+
+ def __init__(self, d=None):
+ self.messages = None
+ if isinstance(d, dict):
+ if 'messages' in d:
+ self.messages = d['messages']
+
+ def read(self, iprot):
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.LIST:
+ self.messages = []
+ (_etype3, _size0) = iprot.readListBegin()
+ for _i4 in xrange(_size0):
+ _elem5 = LogEntry()
+ _elem5.read(iprot)
+ self.messages.append(_elem5)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ oprot.writeStructBegin('Log_args')
+ if self.messages != None:
+ oprot.writeFieldBegin('messages', TType.LIST, 1)
+ oprot.writeListBegin(TType.STRUCT, len(self.messages))
+ for iter6 in self.messages:
+ iter6.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def __str__(self):
+ return str(self.__dict__)
+
+ def __repr__(self):
+ return repr(self.__dict__)
+
+class Log_result:
+
+ def __init__(self, d=None):
+ self.success = None
+ if isinstance(d, dict):
+ if 'success' in d:
+ self.success = d['success']
+
+ def read(self, iprot):
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 0:
+ if ftype == TType.I32:
+ self.success = iprot.readI32();
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ oprot.writeStructBegin('Log_result')
+ if self.success != None:
+ oprot.writeFieldBegin('success', TType.I32, 0)
+ oprot.writeI32(self.success)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def __str__(self):
+ return str(self.__dict__)
+
+ def __repr__(self):
+ return repr(self.__dict__)
+
+
65 lib/py/scribe/ttypes.py
@@ -0,0 +1,65 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+from thrift.protocol.TProtocol import *
+import fb303.ttypes
+
+
+class ResultCode:
+ OK = 0
+ TRY_LATER = 1
+
+class LogEntry:
+
+ def __init__(self, d=None):
+ self.category = None
+ self.message = None
+ if isinstance(d, dict):
+ if 'category' in d:
+ self.category = d['category']
+ if 'message' in d:
+ self.message = d['message']
+
+ def read(self, iprot):
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.category = iprot.readString();
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.message = iprot.readString();
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ oprot.writeStructBegin('LogEntry')
+ if self.category != None:
+ oprot.writeFieldBegin('category', TType.STRING, 1)
+ oprot.writeString(self.category)
+ oprot.writeFieldEnd()
+ if self.message != None:
+ oprot.writeFieldBegin('message', TType.STRING, 2)
+ oprot.writeString(self.message)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def __str__(self):
+ return str(self.__dict__)
+
+ def __repr__(self):
+ return repr(self.__dict__)
+
7 lib/py/setup.py
@@ -0,0 +1,7 @@
+from distutils.core import setup
+
+setup(name='scribe',
+ version='2.0',
+ packages=['scribe'],
+ )
+
127 src/Makefile.am
@@ -0,0 +1,127 @@
+@GLOBAL_HEADER_MK@
+
+@PRODUCT_MK@
+
+all:
+
+# IMPORTANT THINGS TO FOLLOW
+# Do not use relative paths to refer to sources, objects, libs etc not located in the current component dir or its subdirectories.
+# Use $(top_srcdir), $(top_buildir), $(srcdir) which are provided by automake.
+# Example:
+# Use /xxx/yyy OR $(xxx_home)/yyy/.. to refer to paths outside the root of this project.
+# To refer to sources or other input files use $(top_srcdir)/... if outside of current dir but in this project.
+# To refer to all *generated* files, objects, libs etc [ if not in current dir or its subdir], use $(top_builddir)
+# Follow naming conventions for global, common and specific flags.
+# Make build configuratiion and rules readable.
+# Break up into logical sections.
+# Set up command line options, default paths in configure.ac.
+# Generate gen-cpp in current dir not in other locations.
+# Do no create new targets especially if build time is small
+# Default setting is opt mode. Use --disable-opt to turn this off.
+# Default setting is static library . Use --disable-debug to turn on shared.
+
+# Section 1 #########################################################################
+# Set up User defined flags, variables and switches here.
+# It is preferable to define these variables in configure.ac.
+
+# User specified path variables set in configure.ac.
+# thrift_home
+# jvm_lib
+# smc_home
+# fb_home
+# fb303_home
+#
+THRIFT = $(thrift_home)/bin/thrift
+
+# User defined conditionals and conditonal statements set up in configure.ac.
+# FACEBOOK set in configure.ac
+
+if FACEBOOK
+ FB_SOURCES = gen-cpp/ServiceManager_types.cpp gen-cpp/ServiceManager.cpp
+ FB_CPPFLAGS = -I$(fb_home)
+endif
+if DEBUG
+ DEBUG_CPPFLAGS = -DDEBUG_TIMING
+endif
+
+
+# Set libraries external to this component.
+EXTERNAL_LIBS = -L$(thrift_home) -lfb303 -lthrift -lthriftnb
+EXTERNAL_LIBS += -levent -lpthread -lrt
+
+# Section 2 ############################################################################
+# Set common flags recognized by automake.
+# DO NOT USE CPPFLAGS, CXXFLAGS, CFLAGS, LDFLAGS here! Set in configure.ac and|or override on command line.
+# USE flags AM_CXXFLAGS, AM_CFLAGS, AM_CPPFLAGS, AM_LDFLAGS, LDADD in this section.
+
+AM_CPPFLAGS = -I..
+AM_CPPFLAGS += -I$(thrift_home)/include/thrift
+AM_CPPFLAGS += -I$(thrift_home)/include/thrift/fb303
+AM_CPPFLAGS += $(BOOST_CPPFLAGS)
+AM_CPPFLAGS += $(FB_CPPFLAGS) $(DEBUG_CPPFLAGS)
+
+AM_LDFLAGS = $(BOOST_LDFLAGS) $(BOOST_SYSTEM_LIB) $(BOOST_FILESYSTEM_LIB)
+
+# Section 3 #############################################################################
+# GENERATE BUILD RULES
+# Set Program/library specific flags recognized by automake.
+# Use <progname|libname>_<FLAG> to set prog / lib specific flag s
+# foo_CXXFLAGS foo_CPPFLAGS foo_LDFLAGS foo_LDADD
+
+# Static -- multiple libraries can be defined
+if STATIC
+lib_LIBRARIES = libscribe.a
+libscribe_a_SOURCES = gen-cpp/scribe.cpp gen-cpp/scribe_types.cpp gen-cpp/scribe_constants.cpp
+INTERNAL_LIBS = libscribe.a
+endif
+
+# Shared -- multiple libraries can be defined
+if SHARED
+shareddir = lib
+shared_PROGRAMS = libscribe.so
+libscribe_so_SOURCES = gen-cpp/scribe.cpp gen-cpp/scribe_types.cpp
+libscribe_so_CXXFLAGS = $(SHARED_CXXFLAGS)
+libscribe_so_LDFLAGS = $(SHARED_LDFLAGS)
+INTERNAL_LIBS = libscribe.so
+endif
+
+# Binaries -- multiple progs can be defined.
+bin_PROGRAMS = scribed
+scribed_SOURCES = store.cpp store_queue.cpp conf.cpp file.cpp conn_pool.cpp scribe_server.cpp $(FB_SOURCES)
+scribed_LDADD = $(EXTERNAL_LIBS) $(INTERNAL_LIBS)
+
+if SHARED
+scribed_DEPENDENCIES = libscribe.so
+endif
+
+# Section 4 ##############################################################################
+# Set up Thrift specific activity here.
+# We assume that a <name>+types.cpp will always be built from <name>.thrift.
+
+$(eval $(call thrift_template,.,$(srcdir)/../if/scribe.thrift, -I $(thrift_home)/share/ -I $(fb303_home)/share/ --cpp --py --php))
+
+if FACEBOOK
+ $(eval $(call thrift_template,.,$(smc_home)/if/ServiceManager.thrift,--cpp))
+endif
+
+BUILT_SOURCES = thriftstyle
+
+# Section 5 [OPTIONAL] ##################################################################################
+# Create user specific targets [ OPTIONAL]
+
+# Overrides pre-existing target "all". Use only if required.
+# all: <all-special>
+
+# Add to pre-existing target clean
+
+clean-local: clean-common
+
+# Add to pre-existing target all:
+#all-local: scribed
+# @mkdir -p $(top_srcdir)/bin
+# cp $(bin_PROGRAMS) $(top_srcdir)/bin
+
+# Special targets.
+#server-opt : $(BUILT_SOURCES) scribed
+
+@GLOBAL_FOOTER_MK@
74 src/common.h
@@ -0,0 +1,74 @@
+// Copyright (c) 2007-2008 Facebook
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// See accompanying file LICENSE or visit the Scribe site at:
+// http://developers.facebook.com/scribe/
+//
+// @author Bobby Johnson
+// @author Jason Sobel
+
+#ifndef SCRIBE_COMMON_H
+#define SCRIBE_COMMON_H
+
+#include <sstream>
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <string>
+#include <queue>
+#include <vector>
+#include <pthread.h>
+#include <semaphore.h>
+#include <map>
+#include <set>
+#include <stdexcept>
+#include <errno.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <sys/resource.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <boost/shared_ptr.hpp>
+#include <boost/filesystem/operations.hpp>
+
+#include "thrift/protocol/TBinaryProtocol.h"
+#include "thrift/server/TNonblockingServer.h"
+#include "thrift/transport/TSocket.h"
+#include "thrift/transport/TSocketPool.h"
+#include "thrift/transport/TServerSocket.h"
+#include "thrift/transport/TTransportUtils.h"
+#include "thrift/transport/THttpClient.h"
+#include "thrift/transport/TFileTransport.h"
+#include "fb303/FacebookBase.h"
+
+#include "src/gen-cpp/scribe.h"
+
+typedef boost::shared_ptr<scribe::thrift::LogEntry> logentry_ptr_t;
+typedef std::vector<logentry_ptr_t> logentry_vector_t;
+typedef std::vector<std::pair<std::string, int> > server_vector_t;
+
+// For security reasons we can't release everything that's compiled
+// in at facebook. Other users might find this useful as well for
+// integrating to their environment.
+// Things in this file include network based configuration and debug messages
+#ifdef FACEBOOK
+#include "env_facebook.h"
+#else
+#include "env_default.h"
+#endif
+
+#endif // !defined SCRIBE_COMMON_H
186 src/conf.cpp
@@ -0,0 +1,186 @@
+// Copyright (c) 2007-2008 Facebook
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// See accompanying file LICENSE or visit the Scribe site at:
+// http://developers.facebook.com/scribe/
+//
+// @author Bobby Johnson
+// @author Jason Sobel
+
+#include "common.h"
+#include "conf.h"
+
+using namespace boost;
+using namespace std;
+
+StoreConf::StoreConf() {
+}
+
+StoreConf::~StoreConf() {
+}
+
+bool StoreConf::getStore(const std::string& storeName, pStoreConf& _return) {
+ store_conf_map_t::iterator iter = stores.find(storeName);
+ if (iter != stores.end()) {
+ _return = iter->second;
+ return true;
+ } else {
+ return false;
+ }
+}
+
+void StoreConf::getAllStores(std::vector<pStoreConf>& _return) {
+ for (store_conf_map_t::iterator iter = stores.begin(); iter != stores.end(); ++iter) {
+ _return.push_back(iter->second);
+ }
+}
+
+bool StoreConf::getInt(const std::string& intName, long int& _return) {
+ string str;
+ if (getString(intName, str)) {
+ _return = strtol(str.c_str(), NULL, 0);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool StoreConf::getUnsigned(const std::string& intName, unsigned long int& _return) {
+ string str;
+ if (getString(intName, str)) {
+ _return = strtoul(str.c_str(), NULL, 0);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool StoreConf::getString(const std::string& stringName, std::string& _return) {
+ string_map_t::iterator iter = values.find(stringName);
+ if (iter != values.end()) {
+ _return = iter->second;
+ return true;
+ } else {
+ return false;
+ }
+}
+
+void StoreConf::setString(const std::string& stringName, const std::string& value) {
+ values[stringName] = value;
+}
+
+void StoreConf::setUnsigned(const std::string& stringName, unsigned long value) {
+ ostringstream oss;
+ oss << value;
+ setString(stringName, oss.str());
+}
+
+// reads and parses the config data
+void StoreConf::parseConfig(const std::string& filename) {
+
+ queue<string> config_strings;
+
+ if (readConfFile(filename, config_strings)) {
+ LOG_OPER("got configuration data from file <%s>", filename.c_str());
+ } else {
+ std::ostringstream msg;
+ msg << "Failed to open config file <" << filename << ">";
+ throw std::runtime_error(msg.str());
+ }
+
+ parseStore(config_strings, this);
+}
+
+// Side-effects: - removes items from raw_config and adds items to parsed_config
+//
+// Returns true if a valid entry was found
+bool StoreConf::parseStore(queue<string>& raw_config, /*out*/ StoreConf* parsed_config) {
+
+ int store_index = 0; // used to give things named "store" different names
+
+ string line;
+ while (!raw_config.empty()) {
+
+ line = raw_config.front();
+ raw_config.pop();
+
+ int length = line.size();
+ if (0 >= length || line[0] == '#') {
+ continue;
+ }
+ if (line[0] == '<') {
+
+ if (length > 1 && line[1] == '/') {
+ // This is the end of the current store
+ return true;
+ }
+
+ // This is the start of a new store
+ string::size_type pos = line.find('>');
+ if (pos == string::npos) {
+ LOG_OPER("Bad config - line %s has a < but not a >", line.c_str());
+ continue;
+ }
+ string store_name = line.substr(1, pos - 1);
+
+ pStoreConf new_store(new StoreConf);
+ if (parseStore(raw_config, new_store.get())) {
+ if (0 == store_name.compare("store")) {
+ // This is a special case for the top-level stores. They share
+ // the same name, so we append an index to put them in the map
+ std::ostringstream oss;
+ oss << store_index;
+ store_name += oss.str();
+ ++store_index;
+ }
+ if (parsed_config->stores.find(store_name) != parsed_config->stores.end()) {
+ LOG_OPER("Bad config - duplicate store name %s", store_name.c_str());
+ }
+ parsed_config->stores[store_name] = new_store;
+ }
+ } else {
+ string::size_type eq = line.find('=');
+ if (eq == string::npos) {
+ LOG_OPER("Bad config - line %s is missing an =", line.c_str());
+ } else {
+ string arg = line.substr(0, eq);
+ string val = line.substr(eq + 1, string::npos);
+ if (parsed_config->values.find(arg) != parsed_config->values.end()) {
+ LOG_OPER("Bad config - duplicate key %s", arg.c_str());
+ }
+ parsed_config->values[arg] = val;
+ }
+ }
+ }
+ return true;
+}
+
+// reads every line from the file and pushes then onto _return
+// returns false on error
+bool StoreConf::readConfFile(const string& filename, queue<string>& _return) {
+ std::string line;
+ std::ifstream config_file;
+
+ config_file.open(filename.c_str());
+ if (!config_file.good()) {
+ return false;
+ }
+
+ while (std::getline(config_file, line)) {
+ _return.push(line);
+ }
+
+ config_file.close();
+ return true;
+}
74 src/conf.h
@@ -0,0 +1,74 @@
+// Copyright (c) 2007-2008 Facebook
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// See accompanying file LICENSE or visit the Scribe site at:
+// http://developers.facebook.com/scribe/
+//
+// @author Bobby Johnson
+// @author Jason Sobel
+
+#ifndef SCRIBE_CONF_H
+#define SCRIBE_CONF_H
+
+#include <string>
+#include <vector>
+#include <queue>
+#include <iostream>
+#include <fstream>
+
+#include "src/gen-cpp/scribe.h"
+
+/*
+ * This class reads and parses a configuration
+ * describing a hierarchy of store objects.
+ *
+ * It reads a conf file with a proprietary format, although it could
+ * be changed to xml (or anything else that supports hierarchy) by only
+ * changing the code in this class.
+ */
+class StoreConf;
+typedef boost::shared_ptr<StoreConf> pStoreConf;
+typedef std::map<std::string, std::string> string_map_t;
+typedef std::map<std::string, pStoreConf> store_conf_map_t;
+
+class StoreConf {
+
+ public:
+ StoreConf();
+ virtual ~StoreConf();
+
+ // Return value is true if the key exists, and false if it doesn't.
+ // This doesn't check for garbage ints or empty strings.
+ // The return parameter is untouched if the key isn't found.
+ void getAllStores(std::vector<pStoreConf>& _return);
+ bool getStore(const std::string& storeName, pStoreConf& _return);
+ bool getInt(const std::string& intName, long int& _return);
+ bool getUnsigned(const std::string& intName, unsigned long int& _return);
+ bool getString(const std::string& stringName, std::string& _return);
+
+ void setString(const std::string& stringName, const std::string& value);
+ void setUnsigned(const std::string& intName, unsigned long value);
+
+ // Reads configuration from a file and throws an exception if it fails.
+ void parseConfig(const std::string& filename);
+
+ private:
+ string_map_t values;
+ store_conf_map_t stores;
+
+ static bool parseStore(/*in,out*/ std::queue<std::string>& raw_config, /*out*/ StoreConf* parsed_config);
+ bool readConfFile(const std::string& filename, std::queue<std::string>& _return);
+};
+
+#endif //!defined SCRIBE_CONF_H
309 src/conn_pool.cpp
@@ -0,0 +1,309 @@
+// Copyright (c) 2007-2008 Facebook
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//