Many big changes in pants python support along with handy new python …

…libraries this time around!

pants:
  + Fix bang meta-target ~duplicate dep handling
  + Add the ability to override javac compilerargs for compile.
  + Add support for egg dependencies
  + Add pex self-contained binary format generated for python_binary targets
  + Add support for python thrift stub generation
  + Add pylint support
  + Introduce test size layering by convention in BUILD files

jvm:
  + Fix a ZooKeeper client leak when using the get(timeout) API and a timeout occurs
  + Add CompatibilityCodec and support json/thrift binary serialization for ServerSets
  + StateMachine improvements
  + Generalized ObjectSizeCalculator so it correctly works for both 32-bit and 64-bit HotSpot JVM
  + Various checkstyle fixes
  + Add an installable EnumParser for @cmdline Args
  + Dependency upgrades: slf4j 1.6.1 and guice 3.0

python:
  + Restructure twitter python under (src|tests)/python/twitter
  + Add deprecation decorators
  + Add recordio library for structured log reading and writing
  + Add an options library that supports declaring options at point of use instead of in ~__main__
  + Add glog style logging library

(sapling split of 89ae581d728aecc39bde1e8843209a0998cd2b96)

.gitignore

@@ -1,4 +1,6 @@
 *~
+*#
+.#*
 .DS_Store
 .build.properties
 .idea
@@ -10,9 +12,11 @@ codegen/classes/
 dist
 out
 target
+ivy
 
 *.class
 *.iml
 *.ipr
 *.iws
 *.pyc
+*.hprof

3rdparty/python/BUILD

@@ -0,0 +1,44 @@
+# ==================================================================================================
+# Copyright 2011 Twitter, Inc.
+# --------------------------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+# agreements.  See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied.  See the License for the specific language governing permissions and
+# limitations under the License.
+# ==================================================================================================
+
+python_library(
+  name = 'antlr-3.1.3',
+  dependencies = [egg("antlr_python_runtime-3.1.3-py2.6.egg")])
+
+python_library(
+  name = 'py',
+  dependencies = [egg("py-1.4.2-py2.6.egg")])
+
+python_library(
+  name = 'pytest',
+  dependencies = [egg("pytest-2.0.2-py2.6.egg")])
+
+python_library(
+  name = 'python-ldap',
+  dependencies = [egg("python_ldap-*.egg")])
+
+python_library(
+  name = 'mako',
+  dependencies = [egg("Mako-0.4.0-py2.6.egg")])
+
+python_library(
+  name = 'thrift-0.7',
+  dependencies = [egg("Thrift-0.7.0_dev-py2.6.egg")])
+
+python_library(
+  name = 'zookeeper',
+  dependencies = [egg("ZooKeeper-*.egg")])

build-support/pylint/pylint.rc

@@ -0,0 +1,15 @@
+[MASTER]
+
+[BASIC]
+
+[DESIGN]
+
+[SIMILARITIES]
+min-similarity-lines=8
+
+[MISCELLANEOUS]
+notes=FIXME,XXX,TODO
+
+[FORMAT]
+max-line-length=100
+indent-string='  '

pants

@@ -2,17 +2,16 @@
 # ==================================================================================================
 # Copyright 2011 Twitter, Inc.
 # --------------------------------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one or more contributor license
-# agreements.  See the NOTICE file distributed with this work for additional information regarding
-# copyright ownership.  The ASF licenses this file to you under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with the License.  You may
-# obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this work except in compliance with the License.
+# You may obtain a copy of the License in the LICENSE file, or at:
 #
 #  http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software distributed under the
-# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-# express or implied.  See the License for the specific language governing permissions and
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
 # limitations under the License.
 # ==================================================================================================
 
@@ -24,4 +23,4 @@ if [ -z "$ANT_OPTS" ]; then
   export ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m"
 fi
 
-/usr/bin/env python2.6 ${PYTHONPATH}/pants "$@"
+/usr/bin/env python2.6 ${MY_DIR}/src/python/twitter/pants/bin/pants_exe.py "$@"

src/python/twitter/pants/BUILD

@@ -0,0 +1,35 @@
+# ==================================================================================================
+# Copyright 2011 Twitter, Inc.
+# --------------------------------------------------------------------------------------------------
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this work except in compliance with the License.
+# You may obtain a copy of the License in the LICENSE file, or at:
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==================================================================================================
+
+PANTS_SOURCES   = rglobs('*.py') - rglobs('bin/*.py')
+PANTS_GARBAGE   = rglobs('*.pyc') + rglobs('*~')
+PANTS_RESOURCES = rglobs('*') - PANTS_SOURCES - PANTS_GARBAGE
+
+python_library(
+  name = 'pants-lib',
+  dependencies = [
+    pants("3rdparty/python:mako"),
+    pants("src/python/twitter/common")
+  ],
+  sources = PANTS_SOURCES,
+  resources = PANTS_RESOURCES
+)
+
+python_binary(
+  name = 'pants',
+  dependencies = [pants(":pants-lib")],
+  source = 'bin/pants_exe.py',
+)

src/python/twitter/pants/__init__.py

@@ -0,0 +1,152 @@
+# ==================================================================================================
+# Copyright 2011 Twitter, Inc.
+# --------------------------------------------------------------------------------------------------
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this work except in compliance with the License.
+# You may obtain a copy of the License in the LICENSE file, or at:
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==================================================================================================
+
+import os
+import sys
+import fnmatch
+import glob
+
+from twitter.pants.base import Fileset
+from twitter.pants.targets import *
+
+# aliases
+artifact = Artifact
+egg = PythonEgg
+exclude = Exclude
+fancy_pants = Pants
+jar = JarDependency
+jar_library = JarLibrary
+java_library = JavaLibrary
+java_protobuf_library = JavaProtobufLibrary
+java_tests = JavaTests
+java_thrift_library = JavaThriftLibrary
+python_binary = PythonBinary
+python_library = PythonLibrary
+python_thrift_library = PythonThriftLibrary
+python_tests = PythonTests
+python_test_suite = PythonTestSuite
+repo = Repository
+scala_library = ScalaLibrary
+scala_tests = ScalaTests
+
+def globs(*globspecs):
+  """Returns a Fileset that combines the lists of files returned by glob.glob for each globspec."""
+
+  def combine(files, globspec):
+    return files ^ set(glob.glob(globspec))
+  return Fileset(lambda: reduce(combine, globspecs, set()))
+
+def rglobs(*globspecs):
+  """Returns a Fileset that does a recursive scan under the current directory combining the lists of
+  files returned that would be returned by glob.glob for each globspec."""
+
+  root = os.curdir
+  def recursive_globs():
+    for base, dirs, files in os.walk(root):
+      for file in files:
+        path = os.path.relpath(os.path.normpath(os.path.join(base, file)), root)
+        for globspec in globspecs:
+          if fnmatch.fnmatch(path, globspec):
+            yield path
+
+  return Fileset(lambda: set(recursive_globs()))
+
+def has_sources(target):
+  """Returns True if the target has sources."""
+
+  return isinstance(target, TargetWithSources)
+
+def is_exported(target):
+  """Returns True if the target provides an artifact exportable from the repo."""
+
+  return isinstance(target, ExportableJvmLibrary) and target.provides
+
+def is_internal(target):
+  """Returns True if the target is internal to the repo (ie: it might have dependencies)."""
+
+  return isinstance(target, InternalTarget)
+
+def is_jvm(target):
+  """Returns True if the target produces jvm bytecode."""
+
+  return isinstance(target, JvmTarget)
+
+def has_jvm_targets(targets):
+  """Returns true if the given sequence of targets contains at least one jvm target as determined
+  by is_jvm(...)"""
+
+  return len(list(extract_jvm_targets(targets))) > 0
+
+def extract_jvm_targets(targets):
+  """Returns an iterator over the jvm targets the given sequence of targets resolve to.  The given
+  targets can be a mix of types and any non jvm targets (as determined by is_jvm(...) will be
+  filtered out from the returned iterator."""
+
+  for target in targets:
+    if target is None:
+      print >> sys.stderr, 'Warning! Null target!'
+      continue
+    for real_target in target.resolve():
+      if is_jvm(real_target):
+        yield real_target
+
+def is_java(target):
+  """Returns True if the target has or generates java sources."""
+
+  return isinstance(target, JavaLibrary) or (
+    isinstance(target, JavaProtobufLibrary)) or (
+    isinstance(target, JavaTests)) or (
+    isinstance(target, JavaThriftLibrary))
+
+def is_python(target):
+  """Returns True if the target has python sources."""
+
+  return isinstance(target, PythonTarget) or isinstance(target, PythonEgg)
+
+def is_scala(target):
+  """Returns True if the target has scala sources."""
+
+  return isinstance(target, ScalaLibrary) or isinstance(target, ScalaTests)
+
+def is_test(t):
+  """Returns True if the target is comprised of tests."""
+
+  return isinstance(t, JavaTests) or isinstance(t, ScalaTests) or isinstance(t, PythonTests)
+
+# bind this as late as possible
+pants = fancy_pants
+__all__ = (
+  'artifact',
+  'exclude',
+  'egg',
+  'globs',
+  'jar',
+  'jar_library',
+  'java_library',
+  'java_protobuf_library',
+  'java_tests',
+  'java_thrift_library',
+  'pants',
+  'python_binary',
+  'python_library',
+  'python_tests',
+  'python_test_suite',
+  'python_thrift_library',
+  'repo',
+  'rglobs',
+  'scala_library',
+  'scala_tests',
+)

src/python/twitter/pants/ant/__init__.py

@@ -0,0 +1,21 @@
+# ==================================================================================================
+# Copyright 2011 Twitter, Inc.
+# --------------------------------------------------------------------------------------------------
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this work except in compliance with the License.
+# You may obtain a copy of the License in the LICENSE file, or at:
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==================================================================================================
+
+from lib import AntBuilder
+
+__all__ = (
+  AntBuilder,
+)