Permalink
Browse files

initial source import

  • Loading branch information...
0 parents commit 9486041029c89f4b8c4daeb21eec5cb7c80efed0 Nick Kallen committed Apr 12, 2010
Showing with 5,750 additions and 0 deletions.
  1. +8 −0 .gitignore
  2. +4 −0 README.markdown
  3. +22 −0 TODO
  4. +71 −0 ant/bootstrap.xml
  5. +20 −0 ant/clean.xml
  6. +226 −0 ant/compile.xml
  7. +35 −0 ant/docs.xml
  8. +127 −0 ant/package.xml
  9. +55 −0 ant/prepare.xml
  10. +56 −0 ant/test.xml
  11. +39 −0 build.xml
  12. +143 −0 config/development.conf
  13. +143 −0 config/production.conf
  14. +143 −0 config/test.conf
  15. +59 −0 ivy/ivy.xml
  16. +36 −0 ivy/ivysettings.xml
  17. BIN libs/gizzard-1.0.jar
  18. BIN libs/kestrel-1.2.jar
  19. +24 −0 src/main/scala/com/twitter/flockdb/Direction.scala
  20. +8 −0 src/main/scala/com/twitter/flockdb/Edge.scala
  21. +6 −0 src/main/scala/com/twitter/flockdb/EdgeQuery.scala
  22. +193 −0 src/main/scala/com/twitter/flockdb/Edges.scala
  23. +25 −0 src/main/scala/com/twitter/flockdb/ForwardingManager.scala
  24. +8 −0 src/main/scala/com/twitter/flockdb/Priority.scala
  25. +7 −0 src/main/scala/com/twitter/flockdb/QueryTerm.scala
  26. +111 −0 src/main/scala/com/twitter/flockdb/Select.scala
  27. +7 −0 src/main/scala/com/twitter/flockdb/SelectQuery.scala
  28. +27 −0 src/main/scala/com/twitter/flockdb/UuidGenerator.scala
  29. +20 −0 src/main/scala/com/twitter/flockdb/conversions/Edge.scala
  30. +17 −0 src/main/scala/com/twitter/flockdb/conversions/EdgeQuery.scala
  31. +15 −0 src/main/scala/com/twitter/flockdb/conversions/EdgeResults.scala
  32. +30 −0 src/main/scala/com/twitter/flockdb/conversions/ExecuteOperation.scala
  33. +31 −0 src/main/scala/com/twitter/flockdb/conversions/ExecuteOperations.scala
  34. +25 −0 src/main/scala/com/twitter/flockdb/conversions/Priority.scala
  35. +33 −0 src/main/scala/com/twitter/flockdb/conversions/QueryTerm.scala
  36. +26 −0 src/main/scala/com/twitter/flockdb/conversions/SelectOperation.scala
  37. +20 −0 src/main/scala/com/twitter/flockdb/conversions/SelectQuery.scala
  38. +75 −0 src/main/scala/com/twitter/flockdb/jobs/Copy.scala
  39. +88 −0 src/main/scala/com/twitter/flockdb/jobs/multi/Multi.scala
  40. +95 −0 src/main/scala/com/twitter/flockdb/jobs/single/Single.scala
  41. +5 −0 src/main/scala/com/twitter/flockdb/operations/ExecuteOperation.scala
  42. +9 −0 src/main/scala/com/twitter/flockdb/operations/ExecuteOperationType.scala
  43. +5 −0 src/main/scala/com/twitter/flockdb/operations/ExecuteOperations.scala
  44. +6 −0 src/main/scala/com/twitter/flockdb/operations/SelectOperation.scala
  45. +9 −0 src/main/scala/com/twitter/flockdb/operations/SelectOperationType.scala
  46. +36 −0 src/main/scala/com/twitter/flockdb/queries/DifferenceQuery.scala
  47. +72 −0 src/main/scala/com/twitter/flockdb/queries/ExecuteCompiler.scala
  48. +46 −0 src/main/scala/com/twitter/flockdb/queries/IntersectionQuery.scala
  49. +13 −0 src/main/scala/com/twitter/flockdb/queries/Query.scala
  50. +44 −0 src/main/scala/com/twitter/flockdb/queries/SelectCompiler.scala
  51. +24 −0 src/main/scala/com/twitter/flockdb/queries/SeqQuery.scala
  52. +24 −0 src/main/scala/com/twitter/flockdb/queries/SimpleQuery.scala
  53. +28 −0 src/main/scala/com/twitter/flockdb/queries/UnionQuery.scala
  54. +23 −0 src/main/scala/com/twitter/flockdb/queries/WhereInQuery.scala
  55. +65 −0 src/main/scala/com/twitter/flockdb/shards/BlackHoleShard.scala
  56. +13 −0 src/main/scala/com/twitter/flockdb/shards/BlockedShard.scala
  57. +18 −0 src/main/scala/com/twitter/flockdb/shards/MultiShard.scala
  58. +13 −0 src/main/scala/com/twitter/flockdb/shards/ReadOnlyShard.scala
  59. +38 −0 src/main/scala/com/twitter/flockdb/shards/ReadWriteShard.scala
  60. +29 −0 src/main/scala/com/twitter/flockdb/shards/ReplicatingShard.scala
  61. +45 −0 src/main/scala/com/twitter/flockdb/shards/Shard.scala
  62. +481 −0 src/main/scala/com/twitter/flockdb/shards/SqlShard.scala
  63. +15 −0 src/main/scala/com/twitter/flockdb/shards/StatsCollectingShard.scala
  64. +13 −0 src/main/scala/com/twitter/flockdb/shards/WriteOnlyShard.scala
  65. +20 −0 src/main/scala/com/twitter/flockdb/test/Database.scala
  66. +40 −0 src/main/scala/com/twitter/flockdb/test/EdgesReset.scala
  67. +37 −0 src/main/scala/com/twitter/flockdb/test/Reset.scala
  68. +14 −0 src/main/scala/com/twitter/results/Cursor.scala
  69. +4 −0 src/main/scala/com/twitter/results/Page.scala
  70. +80 −0 src/main/scala/com/twitter/results/ResultWindow.scala
  71. +9 −0 src/main/scala/com/twitter/service/flock/ByteSwapper.scala
  72. +19 −0 src/main/scala/com/twitter/service/flock/State.scala
  73. +52 −0 src/main/scala/com/twitter/service/flock/StatsCollectingQuery.scala
  74. +16 −0 src/main/scala/com/twitter/service/flock/conversions/Page.scala
  75. +14 −0 src/main/scala/com/twitter/service/flock/conversions/Results.scala
  76. +101 −0 src/main/thrift/Edges.thrift
  77. +18 −0 src/main/thrift/Flock.thrift
  78. +13 −0 src/test/scala/com/twitter/TestRunner.scala
  79. +13 −0 src/test/scala/com/twitter/flockdb/StaticEdges.scala
  80. +169 −0 src/test/scala/com/twitter/flockdb/integration/EdgesSpec.scala
  81. +86 −0 src/test/scala/com/twitter/flockdb/integration/IntersectionSpec.scala
  82. +105 −0 src/test/scala/com/twitter/flockdb/integration/SelectCompilerSpec.scala
  83. +14 −0 src/test/scala/com/twitter/flockdb/unit/ByteSwapperSpec.scala
  84. +130 −0 src/test/scala/com/twitter/flockdb/unit/CopySpec.scala
  85. +37 −0 src/test/scala/com/twitter/flockdb/unit/DifferenceQuerySpec.scala
  86. +106 −0 src/test/scala/com/twitter/flockdb/unit/EdgesSpec.scala
  87. +262 −0 src/test/scala/com/twitter/flockdb/unit/ExecuteCompilerSpec.scala
  88. +38 −0 src/test/scala/com/twitter/flockdb/unit/IntersectionQuerySpec.scala
  89. +271 −0 src/test/scala/com/twitter/flockdb/unit/JobSpec.scala
  90. +110 −0 src/test/scala/com/twitter/flockdb/unit/SelectCompilerSpec.scala
  91. +73 −0 src/test/scala/com/twitter/flockdb/unit/SimpleQuerySpec.scala
  92. +598 −0 src/test/scala/com/twitter/flockdb/unit/SqlShardSpec.scala
  93. +67 −0 src/test/scala/com/twitter/flockdb/unit/StatsCollectingShardSpec.scala
  94. +32 −0 src/test/scala/com/twitter/flockdb/unit/UnionQuerySpec.scala
  95. +50 −0 src/test/scala/com/twitter/flockdb/unit/WhereInQuerySpec.scala
@@ -0,0 +1,8 @@
+dist/*
+*.log
+flock.ipr
+flock.iws
+target/*
+.DS_Store
+kestrel/*
+*.hprof.txt
@@ -0,0 +1,4 @@
+mysql> CREATE DATABASE edges_test;
+mysql> CREATE DATABASE flock_edges_test;
+
+% ant test -DDB_USERNAME=fixme -DDB_PASSWORD=fixmetoo
22 TODO
@@ -0,0 +1,22 @@
+TODO:
+
+move com.twitter.service.flock.Reset into gizzard
+move com.twitter.results into gizzard or its own package.
+move ByteSwapper into gizzard
+merge in Gizzard's no_reflection branch.
+move StatsCollectingQuery where??
+create a `main` file by copying glock
+create simple exceptionWrappingProxy using new gizzard niceness interface (cf Rowz)
+rename Edges.scala to Flockdb.scala
+figure out where Flock.thrift goes
+move State into com.twitter.flockdb; make a copy for Groups; they're only identical on accident. not the same thing.
+copy the production.conf config from glock (but be careful with passwords!!!!)
+rename Edges.thrift Flock.thrift
+figure out how to package conf file??
+
+then:
+make sure it works in development mode
+use flocker.rb to create a bunch of shards
+use the ruby gem to insert a bunch of data and make a bunch of queries.
+write readme's.
+write blog post.
@@ -0,0 +1,71 @@
+<project xmlns:ivy="antlib:org.apache.ivy.ant">
+
+ <!-- defaults for all projects -->
+ <property name="source.dir" value="${basedir}/src/main" />
+ <property name="test.source.dir" value="${basedir}/src/test" />
+ <property name="target.dir" value="${basedir}/target" />
+
+ <property environment="env" />
+
+ <property name="ivy.install.version" value="2.1.0-rc2" />
+ <property name="ivy.jar.dir" value="${user.home}/.ivy2" />
+ <property name="ivy.jar.file" value="${ivy.jar.dir}/ivy-${ivy.install.version}.jar" />
+ <property name="jsch.install.version" value="0.1.29" />
+ <property name="jsch.jar.file" value="${ivy.jar.dir}/jsch-${jsch.install.version}.jar" />
+
+ <!--
+ download ivy from the web site so that it can be used without being
+ installed. if the file has already been downloaded, we use a rename
+ trick to avoid hitting the website again. (that would be annoying
+ when building offline.)
+ -->
+ <target name="download-ivy" unless="skip.download">
+ <mkdir dir="${ivy.jar.dir}"/>
+ <condition property="ivy.url" value="file:${ivy.jar.file}">
+ <available file="${ivy.jar.file}" />
+ </condition>
+ <property name="ivy.url" value="http://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar" />
+ <get src="${ivy.url}" dest="${ivy.jar.file}.download" usetimestamp="true" />
+ <move file="${ivy.jar.file}.download" tofile="${ivy.jar.file}" />
+
+ <condition property="jsch.url" value="file:${jsch.jar.file}">
+ <available file="${jsch.jar.file}" />
+ </condition>
+ <property name="jsch.url" value="http://repo1.maven.org/maven2/jsch/jsch/${jsch.install.version}/jsch-${jsch.install.version}.jar" />
+ <get src="${jsch.url}" dest="${jsch.jar.file}.download" usetimestamp="true" />
+ <move file="${jsch.jar.file}.download" tofile="${jsch.jar.file}" />
+ </target>
+
+ <!-- import ivy's ant tasks -->
+ <target name="install-ivy" depends="download-ivy">
+ <path id="ivy.lib.path">
+ <fileset dir="${ivy.jar.dir}" includes="ivy-${ivy.install.version}.jar jsch-${jsch.install.version}.jar"/>
+ </path>
+ <taskdef resource="org/apache/ivy/ant/antlib.xml" uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path" />
+ </target>
+
+ <!-- define filename-friendly names for the OS variants -->
+ <condition property="os.libsname" value="osx">
+ <os name="Mac OS X" />
+ </condition>
+ <condition property="os.jni.ext" value="jnilib">
+ <os name="Mac OS X" />
+ </condition>
+ <condition property="os.libsname" value="linux">
+ <os name="Linux" />
+ </condition>
+ <condition property="os.jni.ext" value="so">
+ <os name="Linux" />
+ </condition>
+
+ <!-- where to look for the ivy config -->
+ <property name="ivy.dep.file" value="${basedir}/ivy/ivy.xml" />
+ <property name="ivy.settings.file" value="${basedir}/ivy/ivysettings.xml" />
+
+ <import file="clean.xml" />
+ <import file="prepare.xml" />
+ <import file="compile.xml" />
+ <import file="test.xml" />
+ <import file="docs.xml" />
+ <import file="package.xml" />
+</project>
@@ -0,0 +1,20 @@
+<project xmlns:ivy="antlib:org.apache.ivy.ant">
+
+ <target name="clean" depends="init" description="erase built files and targets">
+ <delete dir="${target.dir}" />
+ <delete dir="${dist.dir}" />
+ <!-- i dont think this is really a good idea: -->
+ <!-- delete dir="${ivy.jar.dir}/cache/${ivy.organisation}" /-->
+ </target>
+
+ <target name="clean-ivy" depends="prepare" description="erase ivy cache of downloaded packages">
+ <ivy:cleancache />
+ </target>
+
+ <target name="clean-jni" depends="prepare" description="clean out any built jni targets" if="build.jni">
+ <ant dir="src/main/jni" target="clean" />
+ </target>
+
+ <target name="distclean" depends="clean, clean-jni, clean-ivy" />
+
+</project>
@@ -0,0 +1,226 @@
+<project xmlns:ivy="antlib:org.apache.ivy.ant">
+
+ <property name="thrift.source.dir" value="${source.dir}/thrift" />
+ <property name="thrift.target.dir" value="${target.dir}/gen-java" />
+
+
+ <!-- generate thrift stubs if necessary -->
+
+ <macrodef name="generate-thrift">
+ <sequential>
+ <pathconvert property="thrift.gen.path">
+ <path location="${ivy.extra.thriftpackage}" />
+ <unpackagemapper from="${basedir}/*" to="${thrift.target.dir}/*" />
+ </pathconvert>
+ <mkdir dir="${thrift.target.dir}" />
+ <property name="thrift.bin" value="thrift" />
+ <apply executable="${thrift.bin}" dest="${thrift.gen.path}" failonerror="true"
+ skipemptyfilesets="true" verbose="true">
+ <arg value="--gen" />
+ <arg value="java" />
+ <arg value="--gen" />
+ <arg value="rb" />
+ <arg value="-o" />
+ <arg value="${target.dir}" />
+ <arg value="" />
+ <fileset dir="${thrift.source.dir}" includes="**/*.thrift" />
+ <mergemapper from="*.thrift" to="${target.dir}/thrift-run" />
+ </apply>
+ <touch file="${target.dir}/thrift-run" />
+ </sequential>
+ </macrodef>
+
+ <macrodef name="compile-generated-thrift">
+ <sequential>
+ <javac srcdir="${thrift.target.dir}" destdir="${target.dir}/classes" deprecation="on">
+ <classpath>
+ <path refid="deps.path" />
+ </classpath>
+ <include name="**/*.java" />
+ </javac>
+ </sequential>
+ </macrodef>
+
+ <target name="compile-thrift" if="build.thrift">
+ <generate-thrift />
+ <compile-generated-thrift />
+ </target>
+
+
+ <!-- if there's any jni, compile it -->
+
+ <target name="check-jni-source" if="build.jni">
+ <!-- tricksy. remove any target files if any non-target files are newer. -->
+ <dependset>
+ <srcfileset dir="src/main/jni" excludes="**/target/*.${os.jni.ext} **/target/*.jar" />
+ <targetfileset dir="src/main/jni" includes="**/target/*.${os.jni.ext} **/target/*.jar" />
+ </dependset>
+ <!-- and then only set build.jni.ok if those targets are "older" than build.xml (ie missing) -->
+ <uptodate property="build.jni.ok" targetfile="src/main/jni/build.xml">
+ <srcfiles dir="src/main/jni" includes="**/target/*.${os.jni.ext} **/target/*.jar" />
+ </uptodate>
+ </target>
+
+ <target name="compile-jni" depends="check-jni-source" if="build.jni.ok">
+ <ant dir="src/main/jni" target="compile" inheritAll="false" />
+ </target>
+
+ <target name="install-jni" depends="compile-jni" if="build.jni">
+ <copy todir="${dist.dir}" flatten="true">
+ <fileset dir="src/main/jni" includes="**/target/*.${os.jni.ext}" />
+ <fileset dir="src/main/jni" includes="**/target/*.jar" />
+ </copy>
+ </target>
+
+ <!-- compile old-skool java -->
+
+ <target name="compile-java" if="build.java">
+ <javac srcdir="${source.dir}/java" destdir="${target.dir}/classes" deprecation="on">
+ <classpath>
+ <path refid="deps.path" />
+ </classpath>
+ <include name="**/*.java" />
+ </javac>
+ </target>
+
+
+ <!-- compile scala -->
+
+ <target name="compile-scala" if="build.scala">
+ <scalac srcdir="${source.dir}/scala" destdir="${target.dir}/classes" force="changed" deprecation="on" unchecked="on">
+ <classpath>
+ <path refid="deps.path" />
+ </classpath>
+ <include name="**/*.scala" />
+ </scalac>
+ </target>
+
+
+ <!-- create properties file with build info -->
+
+ <condition property="use.git" value="yes">
+ <available file=".git" />
+ </condition>
+
+ <condition property="use.hg" value="yes">
+ <available file=".hg" />
+ </condition>
+
+ <condition property="use.svn" value="yes">
+ <available file=".svn" />
+ </condition>
+
+ <target name="find-git-revision" if="use.git">
+ <!-- ask git for the current "head" commit-id, for memoizing inside the built jar -->
+ <exec outputproperty="revision" executable="git" failifexecutionfails="false">
+ <arg value="rev-parse" />
+ <arg value="HEAD" />
+ </exec>
+ <exec executable="git" failifexecutionfails="false">
+ <arg value="rev-parse" />
+ <arg value="HEAD" />
+ <redirector outputproperty="revision-short">
+ <outputfilterchain>
+ <tokenfilter>
+ <filetokenizer />
+ <replaceregex pattern="(.{8}).*" replace="\1"/>
+ </tokenfilter>
+ </outputfilterchain>
+ </redirector>
+ </exec>
+ </target>
+
+ <target name="find-hg-revision" if="use.hg">
+ <!-- ask hg for the current "tip" commit-id, for memoizing inside the built jar -->
+ <exec outputproperty="revision" executable="hg" failifexecutionfails="false">
+ <arg value="tip" />
+ <arg value="--template" />
+ <arg value="{node}" />
+ </exec>
+ <exec executable="hg" failifexecutionfails="false">
+ <arg value="tip" />
+ <arg value="--template" />
+ <arg value="{node}" />
+ <redirector outputproperty="revision-short">
+ <outputfilterchain>
+ <tokenfilter>
+ <filetokenizer />
+ <replaceregex pattern="(.{8}).*" replace="\1"/>
+ </tokenfilter>
+ </outputfilterchain>
+ </redirector>
+ </exec>
+ </target>
+
+ <target name="find-svn-revision" if="use.svn">
+ <!-- ask svnversion for the revisions in the working copy, for memoizing inside the built jar -->
+ <exec outputproperty="revision" executable="svnversion" failifexecutionfails="false"/>
+ <exec outputproperty="revision-short" executable="svnversion" failifexecutionfails="false"/>
+ </target>
+
+ <target name="find-revision" depends="find-git-revision, find-hg-revision, find-svn-revision" />
+
+ <target name="write-build-info" depends="init, find-revision" if="ivy.extra.buildpackage">
+ <tstamp>
+ <format property="build.timestamp.time" pattern="yyyyMMdd-HHmmss" />
+ <format property="build.timestamp.date" pattern="yyyyMMdd" />
+ </tstamp>
+ <pathconvert property="build.properties.path">
+ <path location="${ivy.extra.buildpackage}" />
+ <unpackagemapper from="${basedir}/*" to="${target.dir}/classes/*" />
+ </pathconvert>
+ <propertyfile file="${build.properties.path}/build.properties">
+ <entry key="name" value="${ivy.module}" />
+ <entry key="version" value="${ivy.revision}" />
+ <entry key="build_name" value="${build.timestamp.time}" />
+ <entry key="build_revision" value="${revision}" />
+ </propertyfile>
+ </target>
+
+
+ <!-- copy resources needed by tests and jar -->
+
+ <target name="copy-resources">
+ <copy todir="${dist.dir}/libs" flatten="true">
+ <path refid="deps.path" />
+ </copy>
+ <copy todir="${target.dir}/test-classes/" failonerror="false">
+ <fileset dir="${test.source.dir}/resources" />
+ </copy>
+ <copy todir="${target.dir}/classes/" overwrite="true" failonerror="false">
+ <fileset dir="${source.dir}/resources" />
+ </copy>
+ </target>
+
+ <target name="copy-config" if="copy.config">
+ <copy todir="${dist.dir}/config">
+ <fileset dir="${basedir}/config" />
+ </copy>
+ </target>
+
+ <target name="copy-extra-config" if="config.extra">
+ <copy todir="${dist.dir}/config">
+ <fileset dir="${config.extra}" />
+ </copy>
+ </target>
+
+ <target name="copy-extra-libs" if="libs.extra">
+ <copy todir="${dist.dir}/libs">
+ <path refid="libs.extra" />
+ </copy>
+ </target>
+
+ <target name="copy-extra-dist" if="dist.extra">
+ <copy todir="${dist.dir}">
+ <path refid="dist.extra" />
+ </copy>
+ </target>
+
+ <target name="copy-extra" depends="copy-resources, copy-config, copy-extra-config, copy-extra-libs, copy-extra-dist" />
+
+
+ <target name="compile" depends="prepare, find-source, compile-thrift, check-jni-source, compile-jni,
+ install-jni, compile-java, compile-scala, write-build-info, copy-extra"
+ description="compile java and scala code" />
+
+</project>
@@ -0,0 +1,35 @@
+<project xmlns:ivy="antlib:org.apache.ivy.ant">
+
+ <target name="vscaladoc" depends="prepare" unless="skip.docs">
+ <delete dir="${docs.target.dir}/scaladoc" />
+ <mkdir dir="${docs.target.dir}/scaladoc" />
+ <pathconvert property="doc.sources" pathsep=" ">
+ <fileset dir="${source.dir}" includes="**/*.scala" />
+ </pathconvert>
+ <path id="docs.path">
+ <path refid="bootstrap.path" />
+ <pathelement location="${target.dir}/classes" />
+ </path>
+ <echo message="Building vscaladoc..." />
+ <java classname="org.scala_tools.vscaladoc.Main" fork="true" failonerror="true">
+ <classpath>
+ <path refid="bootstrap.path" />
+ </classpath>
+ <arg value="-classpath" />
+ <arg pathref="docs.path" />
+ <arg value="-d" />
+ <arg value="${docs.target.dir}/scaladoc" />
+ <arg value="-sourcepath" />
+ <arg value="${source.dir}/scala" />
+ <arg value="-windowtitle" />
+ <arg value="${ivy.module} ${ivy.revision}" />
+ <arg value="-doctitle" />
+ <arg value="${ivy.module} ${ivy.revision}" />
+ <arg value="-linksource" />
+ <arg line="${doc.sources}" />
+ </java>
+ </target>
+
+ <target name="docs" depends="prepare,vscaladoc" unless="skip.docs" description="build source documentation" />
+
+</project>
Oops, something went wrong.

0 comments on commit 9486041

Please sign in to comment.