adding the thrift v0.8.0 python module

 wget http://pypi.python.org/packages/source/t/thrift/thrift-0.8.0.tar.gz#md5=edc7e5311e2ae5593cd6ac06fdab3257
 tar xzf thrift-0.8.0.tar.gz
 cd thrift-0.8.0/
 python setup.py build
 cp -r  build/lib.linux-x86_64-2.7/thrift/ ../thrift

and then our Makefile puts this thrift dir as a python module into kba_corpus.tar.gz for use by the mrjob subcorpus_counter.conf

thrift/TSCons.py

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from os import path
+from SCons.Builder import Builder
+
+def scons_env(env, add=''):
+  opath = path.dirname(path.abspath('$TARGET'))
+  lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE'
+  cppbuild = Builder(action = lstr)
+  env.Append(BUILDERS = {'ThriftCpp' : cppbuild})
+
+def gen_cpp(env, dir, file):
+  scons_env(env)
+  suffixes = ['_types.h', '_types.cpp']
+  targets = map(lambda s: 'gen-cpp/' + file + s, suffixes)
+  return env.ThriftCpp(targets, dir+file+'.thrift')

thrift/TSerialization.py

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from protocol import TBinaryProtocol
+from transport import TTransport
+
+def serialize(thrift_object, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()):
+    transport = TTransport.TMemoryBuffer()
+    protocol = protocol_factory.getProtocol(transport)
+    thrift_object.write(protocol)
+    return transport.getvalue()
+
+def deserialize(base, buf, protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()):
+    transport = TTransport.TMemoryBuffer(buf)
+    protocol = protocol_factory.getProtocol(transport)
+    base.read(protocol)
+    return base
+

thrift/Thrift.py

@@ -0,0 +1,154 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import sys
+
+class TType:
+  STOP   = 0
+  VOID   = 1
+  BOOL   = 2
+  BYTE   = 3
+  I08    = 3
+  DOUBLE = 4
+  I16    = 6
+  I32    = 8
+  I64    = 10
+  STRING = 11
+  UTF7   = 11
+  STRUCT = 12
+  MAP    = 13
+  SET    = 14
+  LIST   = 15
+  UTF8   = 16
+  UTF16  = 17
+
+  _VALUES_TO_NAMES = ( 'STOP',
+                      'VOID',
+                      'BOOL',
+                      'BYTE',
+                      'DOUBLE',
+                      None,
+                      'I16',
+                      None,
+                      'I32',
+                      None,
+                       'I64',
+                       'STRING',
+                       'STRUCT',
+                       'MAP',
+                       'SET',
+                       'LIST',
+                       'UTF8',
+                       'UTF16' )
+
+class TMessageType:
+  CALL  = 1
+  REPLY = 2
+  EXCEPTION = 3
+  ONEWAY = 4
+
+class TProcessor:
+
+  """Base class for procsessor, which works on two streams."""
+
+  def process(iprot, oprot):
+    pass
+
+class TException(Exception):
+
+  """Base class for all thrift exceptions."""
+
+  # BaseException.message is deprecated in Python v[2.6,3.0)
+  if (2,6,0) <= sys.version_info < (3,0):
+    def _get_message(self):
+	    return self._message
+    def _set_message(self, message):
+	    self._message = message
+    message = property(_get_message, _set_message)
+
+  def __init__(self, message=None):
+    Exception.__init__(self, message)
+    self.message = message
+
+class TApplicationException(TException):
+
+  """Application level thrift exceptions."""
+
+  UNKNOWN = 0
+  UNKNOWN_METHOD = 1
+  INVALID_MESSAGE_TYPE = 2
+  WRONG_METHOD_NAME = 3
+  BAD_SEQUENCE_ID = 4
+  MISSING_RESULT = 5
+  INTERNAL_ERROR = 6
+  PROTOCOL_ERROR = 7
+
+  def __init__(self, type=UNKNOWN, message=None):
+    TException.__init__(self, message)
+    self.type = type
+
+  def __str__(self):
+    if self.message:
+      return self.message
+    elif self.type == self.UNKNOWN_METHOD:
+      return 'Unknown method'
+    elif self.type == self.INVALID_MESSAGE_TYPE:
+      return 'Invalid message type'
+    elif self.type == self.WRONG_METHOD_NAME:
+      return 'Wrong method name'
+    elif self.type == self.BAD_SEQUENCE_ID:
+      return 'Bad sequence ID'
+    elif self.type == self.MISSING_RESULT:
+      return 'Missing result'
+    else:
+      return 'Default (unknown) TApplicationException'
+
+  def read(self, iprot):
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.STRING:
+          self.message = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.I32:
+          self.type = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    oprot.writeStructBegin('TApplicationException')
+    if self.message != None:
+      oprot.writeFieldBegin('message', TType.STRING, 1)
+      oprot.writeString(self.message)
+      oprot.writeFieldEnd()
+    if self.type != None:
+      oprot.writeFieldBegin('type', TType.I32, 2)
+      oprot.writeI32(self.type)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()

thrift/__init__.py

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+__all__ = ['Thrift', 'TSCons']

thrift/protocol/TBase.py

@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from thrift.Thrift import *
+from thrift.protocol import TBinaryProtocol
+from thrift.transport import TTransport
+
+try:
+  from thrift.protocol import fastbinary
+except:
+  fastbinary = None
+
+class TBase(object):
+  __slots__ = []
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, getattr(self, key))
+              for key in self.__slots__ ]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    if not isinstance(other, self.__class__):
+      return False
+    for attr in self.__slots__:
+      my_val = getattr(self, attr)
+      other_val = getattr(other, attr)
+      if my_val != other_val:
+        return False
+    return True
+
+  def __ne__(self, other):
+    return not (self == other)
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStruct(self, self.thrift_spec)
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStruct(self, self.thrift_spec)
+
+class TExceptionBase(Exception):
+  # old style class so python2.4 can raise exceptions derived from this
+  #  This can't inherit from TBase because of that limitation.
+  __slots__ = []
+
+  __repr__ = TBase.__repr__.im_func
+  __eq__ = TBase.__eq__.im_func
+  __ne__ = TBase.__ne__.im_func
+  read = TBase.read.im_func
+  write = TBase.write.im_func
+