Skip to content

Commit

Permalink
THRIFT-1115 python TBase class for dynamic (de)serialization, and __s…
Browse files Browse the repository at this point in the history 
…lots__ option for memory savings

Patch: Will Pierce

git-svn-id: https://svn.apache.org/repos/asf/thrift/trunk@1169492 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
@bufferoverflow
bufferoverflow committed Sep 11, 2011
1 parent b288050 commit f4eec7a
Show file tree
Hide file tree
Showing 16 changed files with 758 additions and 125 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@
/test/py/Makefile
/test/py/Makefile.in
/test/py/gen-py
/test/py/gen-py-*
/test/py.twisted/Makefile
/test/py.twisted/Makefile.in
/test/py.twisted/_trial_temp/
Expand Down
203 changes: 157 additions & 46 deletions compiler/cpp/src/generate/t_py_generator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,44 @@ class t_py_generator : public t_generator {
iter = parsed_options.find("new_style");
gen_newstyle_ = (iter != parsed_options.end());

iter = parsed_options.find("slots");
gen_slots_ = (iter != parsed_options.end());

iter = parsed_options.find("dynamic");
gen_dynamic_ = (iter != parsed_options.end());

if (gen_dynamic_) {
gen_newstyle_ = 0; // dynamic is newstyle
gen_dynbaseclass_ = "TBase";
gen_dynbaseclass_exc_ = "TExceptionBase";
import_dynbase_ = "from thrift.protocol.TBase import TBase, TExceptionBase\n";
}

iter = parsed_options.find("dynbase");
if (iter != parsed_options.end()) {
gen_dynbase_ = true;
gen_dynbaseclass_ = (iter->second);
}

iter = parsed_options.find("dynexc");
if (iter != parsed_options.end()) {
gen_dynbaseclass_exc_ = (iter->second);
}

iter = parsed_options.find("dynimport");
if (iter != parsed_options.end()) {
gen_dynbase_ = true;
import_dynbase_ = (iter->second);
}

iter = parsed_options.find("twisted");
gen_twisted_ = (iter != parsed_options.end());

iter = parsed_options.find("utf8strings");
gen_utf8strings_ = (iter != parsed_options.end());

copy_options_ = option_string;

if (gen_twisted_){
out_dir_base_ = "gen-py.twisted";
} else {
Expand Down Expand Up @@ -214,17 +246,32 @@ class t_py_generator : public t_generator {
private:

/**
* True iff we should generate new-style classes.
* True if we should generate new-style classes.
*/
bool gen_newstyle_;

/**
* True if we should generate dynamic style classes.
*/
bool gen_dynamic_;

bool gen_dynbase_;
std::string gen_dynbaseclass_;
std::string gen_dynbaseclass_exc_;

std::string import_dynbase_;

bool gen_slots_;

std::string copy_options_;

/**
* True iff we should generate Twisted-friendly RPC services.
* True if we should generate Twisted-friendly RPC services.
*/
bool gen_twisted_;

/**
* True iff strings should be encoded using utf-8.
* True if strings should be encoded using utf-8.
*/
bool gen_utf8strings_;

Expand Down Expand Up @@ -325,13 +372,19 @@ string t_py_generator::render_includes() {
* Renders all the imports necessary to use the accelerated TBinaryProtocol
*/
string t_py_generator::render_fastbinary_includes() {
return
"from thrift.transport import TTransport\n"
"from thrift.protocol import TBinaryProtocol, TProtocol\n"
"try:\n"
" from thrift.protocol import fastbinary\n"
"except:\n"
" fastbinary = None\n";
string hdr = "";
if (gen_dynamic_) {
hdr += std::string(import_dynbase_);
} else {
hdr +=
"from thrift.transport import TTransport\n"
"from thrift.protocol import TBinaryProtocol, TProtocol\n"
"try:\n"
" from thrift.protocol import fastbinary\n"
"except:\n"
" fastbinary = None\n";
}
return hdr;
}

/**
Expand All @@ -343,6 +396,8 @@ string t_py_generator::py_autogen_comment() {
"# Autogenerated by Thrift Compiler (" + THRIFT_VERSION + ")\n" +
"#\n" +
"# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n" +
"#\n" +
"# options string: " + copy_options_ + "\n" +
"#\n";
}

Expand All @@ -351,7 +406,7 @@ string t_py_generator::py_autogen_comment() {
*/
string t_py_generator::py_imports() {
return
string("from thrift.Thrift import *");
string("from thrift.Thrift import TType, TMessageType");
}

/**
Expand Down Expand Up @@ -384,6 +439,7 @@ void t_py_generator::generate_enum(t_enum* tenum) {
f_types_ <<
"class " << tenum->get_name() <<
(gen_newstyle_ ? "(object)" : "") <<
(gen_dynamic_ ? "(" + gen_dynbaseclass_ + ")" : "") <<
":" << endl;
indent_up();
generate_python_docstring(f_types_, tenum);
Expand Down Expand Up @@ -575,12 +631,19 @@ void t_py_generator::generate_py_struct_definition(ofstream& out,
out << std::endl <<
"class " << tstruct->get_name();
if (is_exception) {
out << "(Exception)";
} else if (gen_newstyle_) {
out << "(object)";
if (gen_dynamic_) {
out << "(" << gen_dynbaseclass_exc_ << ")";
} else {
out << "(Exception)";
}
} else {
if (gen_newstyle_) {
out << "(object)";
} else if (gen_dynamic_) {
out << "(" << gen_dynbaseclass_ << ")";
}
}
out <<
":" << endl;
out << ":" << endl;
indent_up();
generate_python_docstring(out, tstruct);

Expand All @@ -606,6 +669,17 @@ void t_py_generator::generate_py_struct_definition(ofstream& out,
TODO(dreiss): Consider making this work for structs with negative tags.
*/

if (gen_slots_) {
indent(out) << "__slots__ = [ " << endl;
indent_up();
for (m_iter = sorted_members.begin(); m_iter != sorted_members.end(); ++m_iter) {
indent(out) << "'" << (*m_iter)->get_name() << "'," << endl;
}
indent_down();
indent(out) << " ]" << endl << endl;

}

// TODO(dreiss): Look into generating an empty tuple instead of None
// for structures with no members.
// TODO(dreiss): Test encoding of structs where some inner structs
Expand Down Expand Up @@ -672,8 +746,10 @@ void t_py_generator::generate_py_struct_definition(ofstream& out,
out << endl;
}

generate_py_struct_reader(out, tstruct);
generate_py_struct_writer(out, tstruct);
if (!gen_dynamic_) {
generate_py_struct_reader(out, tstruct);
generate_py_struct_writer(out, tstruct);
}

// For exceptions only, generate a __str__ method. This is
// because when raised exceptions are printed to the console, __repr__
Expand All @@ -685,31 +761,61 @@ void t_py_generator::generate_py_struct_definition(ofstream& out,
endl;
}

// Printing utilities so that on the command line thrift
// structs look pretty like dictionaries
out <<
indent() << "def __repr__(self):" << endl <<
indent() << " L = ['%s=%r' % (key, value)" << endl <<
indent() << " for key, value in self.__dict__.iteritems()]" << endl <<
indent() << " return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl <<
endl;
if (!gen_slots_) {
// Printing utilities so that on the command line thrift
// structs look pretty like dictionaries
out <<
indent() << "def __repr__(self):" << endl <<
indent() << " L = ['%s=%r' % (key, value)" << endl <<
indent() << " for key, value in self.__dict__.iteritems()]" << endl <<
indent() << " return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl <<
endl;

// Equality and inequality methods that compare by value
out <<
indent() << "def __eq__(self, other):" << endl;
indent_up();
out <<
indent() << "return isinstance(other, self.__class__) and "
"self.__dict__ == other.__dict__" << endl;
indent_down();
out << endl;
// Equality and inequality methods that compare by value
out <<
indent() << "def __eq__(self, other):" << endl;
indent_up();
out <<
indent() << "return isinstance(other, self.__class__) and "
"self.__dict__ == other.__dict__" << endl;
indent_down();
out << endl;

out <<
indent() << "def __ne__(self, other):" << endl;
indent_up();
out <<
indent() << "return not (self == other)" << endl;
indent_down();
out <<
indent() << "def __ne__(self, other):" << endl;
indent_up();

out <<
indent() << "return not (self == other)" << endl;
indent_down();
} else if (!gen_dynamic_) {
// no base class available to implement __eq__ and __repr__ and __ne__ for us
// so we must provide one that uses __slots__
out <<
indent() << "def __repr__(self):" << endl <<
indent() << " L = ['%s=%r' % (key, getattr(self, key))" << endl <<
indent() << " for key in self.__slots__]" << endl <<
indent() << " return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl <<
endl;

// Equality method that compares each attribute by value and type, walking __slots__
out <<
indent() << "def __eq__(self, other):" << endl <<
indent() << " if not isinstance(other, self.__class__):" << endl <<
indent() << " return False" << endl <<
indent() << " for attr in self.__slots__:" << endl <<
indent() << " my_val = getattr(self, attr)" << endl <<
indent() << " other_val = getattr(other, attr)" << endl <<
indent() << " if my_val != other_val:" << endl <<
indent() << " return False" << endl <<
indent() << " return True" << endl <<
endl;

out <<
indent() << "def __ne__(self, other):" << endl <<
indent() << " return not (self == other)" << endl <<
endl;
}
indent_down();
}

Expand Down Expand Up @@ -984,7 +1090,7 @@ void t_py_generator::generate_service_interface(t_service* tservice) {
} else {
if (gen_twisted_) {
extends_if = "(Interface)";
} else if (gen_newstyle_) {
} else if (gen_newstyle_ || gen_dynamic_) {
extends_if = "(object)";
}
}
Expand Down Expand Up @@ -1031,8 +1137,8 @@ void t_py_generator::generate_service_client(t_service* tservice) {
extends_client = extends + ".Client, ";
}
} else {
if (gen_twisted_ && gen_newstyle_) {
extends_client = "(object)";
if (gen_twisted_ && (gen_newstyle_ || gen_dynamic_)) {
extends_client = "(object)";
}
}

Expand Down Expand Up @@ -2388,6 +2494,11 @@ string t_py_generator::type_to_spec_args(t_type* ttype) {
THRIFT_REGISTER_GENERATOR(py, "Python",
" new_style: Generate new-style classes.\n" \
" twisted: Generate Twisted-friendly RPC services.\n" \
" utf8strings: Encode/decode strings using utf8 in the generated code.\n"
)
" utf8strings: Encode/decode strings using utf8 in the generated code.\n" \
" slots: Generate code using slots for instance members.\n" \
" dynamic: Generate dynamic code, less code generated but slower.\n" \
" dynbase=CLS Derive generated classes from class CLS instead of TBase.\n" \
" dynexc=CLS Derive generated exceptions from CLS instead of TExceptionBase.\n" \
" dynimport='from foo.bar import CLS'\n" \
" Add an import line to generated code to find the dynbase class.\n")

19 changes: 19 additions & 0 deletions lib/py/src/Thrift.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,25 @@ class TType:
UTF8 = 16
UTF16 = 17

_VALUES_TO_NAMES = ( 'STOP',
'VOID',
'BOOL',
'BYTE',
'DOUBLE',
None,
'I16',
None,
'I32',
None,
'I64',
'STRING',
'STRUCT',
'MAP',
'SET',
'LIST',
'UTF8',
'UTF16' )

class TMessageType:
CALL = 1
REPLY = 2
Expand Down
Loading

0 comments on commit f4eec7a

Please sign in to comment.