google_appengine/bulkloader.py google_appengine/google/appengine/api/labs/__init__.py google_appengine/google/appengine/api/labs/taskqueue/__init__.py google_appengine/google/appengine/api/labs/taskqueue/taskqueue.py google_appengine/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py google_appengine/google/appengine/api/labs/taskqueue/taskqueue_stub.py google_appengine/google/appengine/api/lib_config.py google_appengine/google/appengine/api/namespace_manager/__init__.py google_appengine/google/appengine/api/queueinfo.py google_appengine/google/appengine/api/quota.py google_appengine/google/appengine/api/xmpp/__init__.py google_appengine/google/appengine/api/xmpp/xmpp_service_pb.py google_appengine/google/appengine/api/xmpp/xmpp_service_stub.py google_appengine/google/appengine/datastore/action_pb.py google_appengine/google/appengine/datastore/datastore_v3_pb.py google_appengine/google/appengine/dist/__init__.py google_appengine/google/appengine/dist/_library.py google_appengine/google/appengine/dist/ftplib.py google_appengine/google/appengine/dist/httplib.py google_appengine/google/appengine/dist/neo_cgi.py google_appengine/google/appengine/dist/py_imp.py google_appengine/google/appengine/dist/py_zipimport.py google_appengine/google/appengine/dist/select.py google_appengine/google/appengine/dist/socket.py google_appengine/google/appengine/dist/subprocess.py google_appengine/google/appengine/dist/tempfile.py google_appengine/google/appengine/ext/admin/templates/cron.html google_appengine/google/appengine/ext/admin/templates/css/cron.css google_appengine/google/appengine/ext/admin/templates/css/inboundmail.css google_appengine/google/appengine/ext/admin/templates/css/queues.css google_appengine/google/appengine/ext/admin/templates/css/tasks.css google_appengine/google/appengine/ext/admin/templates/css/xmpp.css google_appengine/google/appengine/ext/admin/templates/inboundmail.html google_appengine/google/appengine/ext/admin/templates/js/multipart_form_data.js google_appengine/google/appengine/ext/admin/templates/js/rfc822_date.js google_appengine/google/appengine/ext/admin/templates/js/webhook.js google_appengine/google/appengine/ext/admin/templates/queues.html google_appengine/google/appengine/ext/admin/templates/tasks.html google_appengine/google/appengine/ext/admin/templates/xmpp.html google_appengine/google/appengine/ext/db/stats.py google_appengine/google/appengine/ext/deferred/__init__.py google_appengine/google/appengine/ext/deferred/deferred.py google_appengine/google/appengine/ext/ereporter/__init__.py google_appengine/google/appengine/ext/ereporter/ereporter.py google_appengine/google/appengine/ext/ereporter/report_generator.py google_appengine/google/appengine/ext/ereporter/templates/report.html google_appengine/google/appengine/ext/key_range/__init__.py google_appengine/google/appengine/ext/remote_api/remote_api_stub.py google_appengine/google/appengine/ext/remote_api/throttle.py google_appengine/google/appengine/ext/webapp/mail_handlers.py google_appengine/google/appengine/ext/webapp/xmpp_handlers.py google_appengine/google/appengine/tools/adaptive_thread_pool.py google_appengine/google/appengine/tools/bulkloader.py google_appengine/google/appengine/tools/remote_api_shell.py google_appengine/google/appengine/tools/requeue.py google_appengine/google/net/proto/message_set.py google_appengine/lib/django/PKG-INFO google_appengine/remote_api_shell.py @@ -1,7 +1,209 @@ Copyright 2008 Google Inc. All rights reserved. -App Engine SDK - Release Notes +App Engine Python SDK - Release Notes + +Version 1.2.6 - September 17, 2009 +================================== + - Added incoming email support. + http://code.google.com/p/googleappengine/issues/detail?id=165 + - Remote API now supports XMPP and task queues. + - The default for all handlers is now secure: optional. Users can + now access all pages via SSL unless explicitly disallowed. + - Remote API now supports HTTPS. + http://code.google.com/p/googleappengine/issues/detail?id=1461 + - Appcfg now uses https by default. + http://code.google.com/p/googleappengine/issues/detail?id=794 + - Appcfg.py now supports the --application and --version flags to + override the values specified in app.yaml. + http://code.google.com/p/googleappengine/issues/detail?id=670 + - GQL now supports '= NULL' queries. + - The db.Model constructor now supports explicitly setting a key + (and thus an id) for a Model instance. + - New Datastore stats api. Stats are also visible in the admin console. + - Bulkloader dump and restore now supports restoring to a different + app id and restoring numeric keys. + + +Version 1.2.5 - August 13, 2009 +=============================== + - The Windows Python SDK now includes a GUI launcher, similar to the Mac SDK. + - Added XMPP support. + http://code.google.com/appengine/docs/python/xmpp + http://code.google.com/p/googleappengine/issues/detail?id=231 + - Datastore now supports multiple writes to the same entity within a + transaction. + - Datastore entity key names can now start with a digit. + http://code.google.com/p/googleappengine/issues/detail?id=1352 + - Datastore now supports ancestor + kind queries without a composite index + http://code.google.com/p/googleappengine/issues/detail?id=1003 + - Bulkloader now supports configurationless dump and restore with new + --dump and --restore options. + - Bulkloader now supports a --dry_run flag to testing data prior to uploading. + - Appcfg.py now allows specifying any end date for request_logs. + - Urlfetch now allows setting the Referer header. + http://code.google.com/p/googleappengine/issues/detail?id=445 + - Urlfetch stub now correctly handles HEAD requests. + http://code.google.com/p/googleappengine/issues/detail?id=866 + - New remote_api_shell tool for interactive remote_api operations. + - New google.ext.ereporter module to collect and email exception reports. + - New google.ext.deferred module to execute ad-hoc tasks on the Task Queue. + +Version 1.2.4 - July 16, 2009 +============================= + - Added support for kindless queries, ie. transaction descendant queries. + http://code.google.com/p/googleappengine/issues/detail?id=913 + - Composite indexes no longer required for certain types of key queries. + - Improved exception reporting in the bulkloader. + - Datastore transaction RPC sent at beginning of transaction rather than + upon first Datastore request. + - PolyModel supports keys_only query. + http://code.google.com/p/googleappengine/issues/detail?id=1630 + - Remote API supports more API's (Images, Memcache and URLFetch). + http://code.google.com/p/googleappengine/issues/detail?id=1596 + - Remote API shell. + - Support for multiple inheritance for Model and PolyModel. + - Enhancement to SearchableModel allowing multiple properties to be + indexed. + - Various code quality improvements. + +Version 1.2.3 - June 1, 2009 +============================ + + - Task Queue support available as google.appengine.api.labs.taskqueue. + http://code.google.com/appengine/docs/python/taskqueue + - Django 1.0 support. You must install Django locally on your machine + for the SDK but no longer need to upload it to App Engine. + from google.appengine.dist import use_library + use_library('django', '1.0') + http://code.google.com/p/googleappengine/issues/detail?id=872 + - Urlfetch supports asynchronous requests. + http://code.google.com/p/googleappengine/issues/detail?id=958 + - Urlfetch in SDK now matches App Engine more closely: + By default, it now sets the referer header, does not set the Accept + header, and sets Accept-Encoding to gzip. + http://code.google.com/p/googleappengine/issues/detail?id=970 + - Fixed issue with httplib and absolute URLs. + http://code.google.com/p/googleappengine/issues/detail?id=1311 + - Memcache key length is no longer restricted to 250 bytes: longer keys + will be replaced with a hash of the key. + - Datastore ancestor queries now work within transactions. + - Datastore transactions in SDK now snapshot on the first operation so they + do not see writes made during the transaction. Matches App Engine. + +Version 1.2.2 - April 22, 2009 +============================== + + - New quota API which returns the CPU usage of the current request. + from google.appengine.api import quota + cpu_usage_so_far = quota.get_request_cpu_usage() + - Urlfetch fetch now has support for user configurable deadlines. + http://code.google.com/p/googleappengine/issues/detail?id=79 + - Urlfetch in the SDK allows the Accept-Encoding header to match App Engine. + http://code.google.com/p/googleappengine/issues/detail?id=1071 + - urllib now supports HTTPS in addition to HTTP + http://code.google.com/p/googleappengine/issues/detail?id=1156 + - Datastore indexes on single properties can now be disabled by setting + indexed=False on the property constructor. + - Datastore now supports Key-only queries, using either SELECT __key__ or + or db.Query(Model, keys_only=True) + - Fixed issues with Datastore IN filters and sorting: sort order is now + correct, and can be used with __key__. + http://code.google.com/p/googleappengine/issues/detail?id=1100 + http://code.google.com/p/googleappengine/issues/detail?id=1016 + - Cron supports additional time specification formats. + http://code.google.com/p/googleappengine/issues/detail?id=1261 + - Fixed an issue in the dev_appserver admin console datastore viewer + (/_ah/admin/datastore) with sorting columns containing None types. + http://code.google.com/p/googleappengine/issues/detail?id=1007 + - Bulk Loader improvements: New appcfg download_data command. + Better backoff support and debugging output for long requests. + - New --vhost flag on appcfg.py request_logs command to select logs for + a particular host. + - Python _ast module is now available for import + http://code.google.com/p/googleappengine/issues/detail?id=779 + - Fixed issue with the color argument of the Images API composite method. + +Version 1.2.1 - April 13, 2009 +============================= + + - Stable, unique IDs for User objects. The Users service now + provides a unique user_id for each user that stays the same even + if a user changes her email address. + http://code.google.com/p/googleappengine/issues/detail?id=1019 + - The Images API now supports compositing images and calculating + a color histogram for an image. + - New allowed mail attachment types: ics, vcf + http://code.google.com/p/googleappengine/issues/detail?id=494 + - Urlfetch requests can now set the User-Agent header. + http://code.google.com/p/googleappengine/issues/detail?id=342 + - An App Engine-specific version of the Python PyCrypto cryptography + library is now available. Learn more at + http://code.google.com/appengine/docs/python/tools/libraries.html + - The bulk loader configuration format has changed.to allow non-CSV + input. This change is not backwards compatible, so you will need to + update your code. + An early release of the bulk downloader is also now available in + bulkloader.py. Learn more about these changes at: + http://code.google.com/appengine/docs/python/tools/uploadingdata.html + - Fixed parsing of unicode GQL queries. + http://code.google.com/p/googleappengine/issues/detail?id=1105 + - Fixed dev_appserver security restrictions for os.path + http://code.google.com/p/googleappengine/issues/detail?id=1068 + - Fixed Reply-To header set in emails sent from dev_appserver. + http://code.google.com/p/googleappengine/issues/detail?id=1017 + + +Version 1.2.0 - March 24, 2009 +============================== + - Cron support. Appcfg.py will upload the schedule to App Engine. + The dev_appserver console at /_ah/admin describes your schedule but does + not automatically run scheduled jobs. Learn more at + http://code.google.com/appengine/docs/python/config/cron.html + - New allow_skipped_files flag in dev_appserver to allow it to read files + which are not available in App Engine. + http://code.google.com/p/googleappengine/issues/detail?id=550 + - New upload_data command in appcfg to run the bulk uploader. + http://code.google.com/appengine/docs/python/tools/uploadingdata.html + +Version 1.1.9 - February 2, 2009 +================================ + + - HTTP Request and Response limit raised to 10MB from 1MB. + Note that API call limits remain at 1MB. + http://code.google.com/p/googleappengine/issues/detail?id=78 + - urllib and urllib2 now available, implemented using urlfetch. + Also adds additional stubs which may enable other modules. + http://code.google.com/p/googleappengine/issues/detail?id=61 + http://code.google.com/p/googleappengine/issues/detail?id=68 + http://code.google.com/p/googleappengine/issues/detail?id=572 + http://code.google.com/p/googleappengine/issues/detail?id=821 + - Early release of a new data bulk upload tool, bulkloader.py + http://code.google.com/appengine/docs/python/tools/uploadingdata.html + - New remote_api for datastore at google.appengine.ext.remote_api + - Single property descending indexes are automatically generated. + - Added db.Query support for IN and != operators. + http://code.google.com/p/googleappengine/issues/detail?id=751 + - Fixed issue where gql date/time parsing could not handle Unicode strings. + - Fixed issue with db model instance key() returning the wrong key for + unsaved instances with parent as key + http://code.google.com/p/googleappengine/issues/detail?id=883 + - New run_in_transaction_custom_retries method for datastore. + - Fixed issue with relative dev_appserver datastore and history paths. + http://code.google.com/p/googleappengine/issues/detail?id=845 + - Static files and skipped files are not readable in dev_appserver, to match + the behavior on App Engine. + http://code.google.com/p/googleappengine/issues/detail?id=550 + - Images API allows multiple transforms of the same type in one request. A + limit of 10 total transforms per request has been added. + - PIL import will work with both PIL.Image and Image. + http://code.google.com/p/googleappengine/issues/detail?id=929 + - Fixed an issue with sending email in dev_appserver when the application + code changed. + http://code.google.com/p/googleappengine/issues/detail?id=182 + - Memcache counters (incr/decr) do nothing on non positive integers to match + the behavior on App Engine. + http://code.google.com/p/googleappengine/issues/detail?id=918 Version 1.1.8 - January 7, 2008 ================================= google_appengine/RELEASE_NOTES @@ -1,3 +1,3 @@ -release: "1.1.8" -timestamp: 1231809440 +release: "1.2.6" +timestamp: 1253219443 api_versions: ['1'] google_appengine/VERSION @@ -48,12 +48,12 @@ SCRIPT_EXCEPTIONS = { "dev_appserver.py" : "dev_appserver_main.py" } -def run_file(file_path, globals_): +def run_file(file_path, globals_, script_dir=SCRIPT_DIR): """Execute the file at the specified path with the passed-in globals.""" sys.path = EXTRA_PATHS + sys.path script_name = os.path.basename(file_path) script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name) - script_path = os.path.join(SCRIPT_DIR, script_name) + script_path = os.path.join(script_dir, script_name) execfile(script_path, globals_) if __name__ == '__main__': google_appengine/appcfg.py @@ -48,12 +48,12 @@ SCRIPT_EXCEPTIONS = { "dev_appserver.py" : "dev_appserver_main.py" } -def run_file(file_path, globals_): +def run_file(file_path, globals_, script_dir=SCRIPT_DIR): """Execute the file at the specified path with the passed-in globals.""" sys.path = EXTRA_PATHS + sys.path script_name = os.path.basename(file_path) script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name) - script_path = os.path.join(SCRIPT_DIR, script_name) + script_path = os.path.join(script_dir, script_name) execfile(script_path, globals_) if __name__ == '__main__': google_appengine/bulkload_client.py @@ -48,12 +48,12 @@ SCRIPT_EXCEPTIONS = { "dev_appserver.py" : "dev_appserver_main.py" } -def run_file(file_path, globals_): +def run_file(file_path, globals_, script_dir=SCRIPT_DIR): """Execute the file at the specified path with the passed-in globals.""" sys.path = EXTRA_PATHS + sys.path script_name = os.path.basename(file_path) script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name) - script_path = os.path.join(SCRIPT_DIR, script_name) + script_path = os.path.join(script_dir, script_name) execfile(script_path, globals_) if __name__ == '__main__': google_appengine/dev_appserver.py @@ -36,8 +36,9 @@ class StringProto(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -87,18 +88,21 @@ class StringProto(ProtocolBuffer.ProtocolMessage): if self.has_value_: res+=prefix+("value: %s\n" % self.DebugFormatString(self.value_)) return res - kvalue = 1 - _TEXT = ( - "ErrorCode", - "value", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kvalue = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "value", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -116,8 +120,9 @@ class Integer32Proto(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0 def has_value(self): return self.has_value_ @@ -167,18 +172,21 @@ class Integer32Proto(ProtocolBuffer.ProtocolMessage): if self.has_value_: res+=prefix+("value: %s\n" % self.DebugFormatInt32(self.value_)) return res - kvalue = 1 - _TEXT = ( - "ErrorCode", - "value", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kvalue = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "value", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -196,8 +204,9 @@ class Integer64Proto(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0 def has_value(self): return self.has_value_ @@ -247,18 +256,21 @@ class Integer64Proto(ProtocolBuffer.ProtocolMessage): if self.has_value_: res+=prefix+("value: %s\n" % self.DebugFormatInt64(self.value_)) return res - kvalue = 1 - _TEXT = ( - "ErrorCode", - "value", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kvalue = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "value", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -276,8 +288,9 @@ class BoolProto(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0 def has_value(self): return self.has_value_ @@ -326,18 +339,21 @@ class BoolProto(ProtocolBuffer.ProtocolMessage): if self.has_value_: res+=prefix+("value: %s\n" % self.DebugFormatBool(self.value_)) return res - kvalue = 1 - _TEXT = ( - "ErrorCode", - "value", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kvalue = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "value", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -355,8 +371,9 @@ class DoubleProto(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = 0.0 + if self.has_value_: + self.has_value_ = 0 + self.value_ = 0.0 def has_value(self): return self.has_value_ @@ -405,18 +422,105 @@ class DoubleProto(ProtocolBuffer.ProtocolMessage): if self.has_value_: res+=prefix+("value: %s\n" % self.DebugFormat(self.value_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kvalue = 1 - _TEXT = ( - "ErrorCode", - "value", - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "value", + }, 1) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.DOUBLE, + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.DOUBLE, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) - ) + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class BytesProto(ProtocolBuffer.ProtocolMessage): + has_value_ = 0 + value_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def value(self): return self.value_ + + def set_value(self, x): + self.has_value_ = 1 + self.value_ = x + + def clear_value(self): + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" + + def has_value(self): return self.has_value_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_value()): self.set_value(x.value()) + + def Equals(self, x): + if x is self: return 1 + if self.has_value_ != x.has_value_: return 0 + if self.has_value_ and self.value_ != x.value_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_value_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: value not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(len(self.value_)) + return n + 1 + + def Clear(self): + self.clear_value() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putPrefixedString(self.value_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_value(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_value_: res+=prefix+("value: %s\n" % self.DebugFormatString(self.value_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kvalue = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "value", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -460,15 +564,19 @@ class VoidProto(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" -__all__ = ['StringProto','Integer32Proto','Integer64Proto','BoolProto','DoubleProto','VoidProto'] +__all__ = ['StringProto','Integer32Proto','Integer64Proto','BoolProto','DoubleProto','BytesProto','VoidProto'] google_appengine/google/appengine/api/api_base_pb.py @@ -37,7 +37,7 @@ class RPC(object): FINISHING = 2 def __init__(self, package=None, call=None, request=None, response=None, - callback=None, stub=None): + callback=None, deadline=None, stub=None): """Constructor for the RPC object. All arguments are optional, and simply set members on the class. @@ -49,6 +49,8 @@ class RPC(object): request: ProtocolMessage instance, appropriate for the arguments response: ProtocolMessage instance, appropriate for the response callback: callable, called when call is complete + deadline: A double specifying the deadline for this call as the number of + seconds from the current time. Ignored if non-positive. stub: APIProxyStub instance, used in default _WaitImpl to do real call """ self.__exception = None @@ -60,10 +62,12 @@ class RPC(object): self.request = request self.response = response self.callback = callback + self.deadline = deadline self.stub = stub + self.cpu_usage_mcycles = 0 def MakeCall(self, package=None, call=None, request=None, response=None, - callback=None): + callback=None, deadline=None): """Makes an asynchronous (i.e. non-blocking) API call within the specified package for the specified call method. @@ -81,11 +85,11 @@ class RPC(object): self.call = call or self.call self.request = request or self.request self.response = response or self.response + self.deadline = deadline or self.deadline assert self.__state is RPC.IDLE, ('RPC for %s.%s has already been started' % (self.package, self.call)) assert self.callback is None or callable(self.callback) - self._MakeCallImpl() def Wait(self): google_appengine/google/appengine/api/apiproxy_rpc.py @@ -21,6 +21,7 @@ Classes/variables/functions defined here: APIProxyStubMap: container of APIProxy stubs. apiproxy: global instance of an APIProxyStubMap. MakeSyncCall: APIProxy entry point. + UserRPC: User-visible class wrapping asynchronous RPCs. """ @@ -30,6 +31,9 @@ Classes/variables/functions defined here: import inspect import sys +from google.appengine.api import apiproxy_rpc + + def CreateRPC(service): """Creates a RPC instance for the given service. @@ -48,8 +52,8 @@ def CreateRPC(service): """ stub = apiproxy.GetStub(service) assert stub, 'No api proxy found for service "%s"' % service - assert hasattr(stub, 'CreateRPC'), ('The service "%s" doesn\'t have ' + - 'a CreateRPC method.' % service) + assert hasattr(stub, 'CreateRPC'), (('The service "%s" doesn\'t have ' + + 'a CreateRPC method.') % service) return stub.CreateRPC() @@ -107,7 +111,10 @@ class ListOfHooks(object): unique_key = (key, inspect.getmodule(function)) if unique_key in self.__unique_keys: return False - self.__content.insert(index, (key, function, service)) + num_args = len(inspect.getargspec(function)[0]) + if (inspect.ismethod(function)): + num_args -= 1 + self.__content.insert(index, (key, function, service, num_args)) self.__unique_keys.add(unique_key) return True @@ -146,21 +153,35 @@ class ListOfHooks(object): self.__content = [] self.__unique_keys = set() - def Call(self, service, call, request, response): + def Call(self, service, call, request, response, rpc=None, error=None): """Invokes all hooks in this collection. + NOTE: For backwards compatibility, if error is not None, hooks + with 4 or 5 arguments are *not* called. This situation + (error=None) only occurs when the RPC request raised an exception; + in the past no hooks would be called at all in that case. + Args: service: string representing which service to call call: string representing which function to call request: protocol buffer for the request response: protocol buffer for the response + rpc: optional RPC used to make this call + error: optional Exception instance to be passed as 6th argument """ - for key, function, srv in self.__content: + for key, function, srv, num_args in self.__content: if srv is None or srv == service: - function(service, call, request, response) + if num_args == 6: + function(service, call, request, response, rpc, error) + elif error is not None: + pass + elif num_args == 5: + function(service, call, request, response, rpc) + else: + function(service, call, request, response) -class APIProxyStubMap: +class APIProxyStubMap(object): """Container of APIProxy stubs for more convenient unittesting. Stubs may be either trivial implementations of APIProxy services (e.g. @@ -202,7 +223,7 @@ class APIProxyStubMap: service: string stub: stub """ - assert not self.__stub_map.has_key(service) + assert not self.__stub_map.has_key(service), repr(service) self.__stub_map[service] = stub if service == 'datastore': @@ -236,9 +257,225 @@ class APIProxyStubMap: """ stub = self.GetStub(service) assert stub, 'No api proxy found for service "%s"' % service - self.__precall_hooks.Call(service, call, request, response) - stub.MakeSyncCall(service, call, request, response) - self.__postcall_hooks.Call(service, call, request, response) + if hasattr(stub, 'CreateRPC'): + rpc = stub.CreateRPC() + self.__precall_hooks.Call(service, call, request, response, rpc) + try: + rpc.MakeCall(service, call, request, response) + rpc.Wait() + rpc.CheckSuccess() + except Exception, err: + self.__postcall_hooks.Call(service, call, request, response, rpc, err) + raise + else: + self.__postcall_hooks.Call(service, call, request, response, rpc) + else: + self.__precall_hooks.Call(service, call, request, response) + try: + stub.MakeSyncCall(service, call, request, response) + except Exception, err: + self.__postcall_hooks.Call(service, call, request, response, None, err) + raise + else: + self.__postcall_hooks.Call(service, call, request, response) + + +class UserRPC(object): + """Wrapper class for asynchronous RPC. + + Simplest low-level usage pattern: + + rpc = UserRPC('service', [deadline], [callback]) + rpc.make_call('method', request, response) + . + . + . + rpc.wait() + rpc.check_success() + + However, a service module normally provides a wrapper so that the + typical usage pattern becomes more like this: + + from google.appengine.api import service + rpc = service.create_rpc([deadline], [callback]) + service.make_method_call(rpc, [service-specific-args]) + . + . + . + rpc.wait() + result = rpc.get_result() + + The service.make_method_call() function sets a service- and method- + specific hook function that is called by rpc.get_result() with the + rpc object as its first argument, and service-specific value as its + second argument. The hook function should call rpc.check_success() + and then extract the user-level result from the rpc.result + protobuffer. Additional arguments may be passed from + make_method_call() to the get_result hook via the second argument. + """ + + __method = None + __get_result_hook = None + __user_data = None + __postcall_hooks_called = False + + def __init__(self, service, deadline=None, callback=None): + """Constructor. + + Args: + service: The service name. + deadline: Optional deadline. Default depends on the implementation. + callback: Optional argument-less callback function. + """ + self.__service = service + self.__rpc = CreateRPC(service) + self.__rpc.deadline = deadline + self.__rpc.callback = callback + + @property + def service(self): + """Return the service name.""" + return self.__service + + @property + def method(self): + """Return the method name.""" + return self.__method + + @property + def deadline(self): + """Return the deadline, if set explicitly (otherwise None).""" + return self.__rpc.deadline + + def __get_callback(self): + """Return the callback attribute, a function without arguments. + + This attribute can also be assigned to. For example, the + following code calls some_other_function(rpc) when the RPC is + complete: + + rpc = service.create_rpc() + rpc.callback = lambda: some_other_function(rpc) + service.make_method_call(rpc) + rpc.wait() + """ + return self.__rpc.callback + def __set_callback(self, callback): + """Set the callback function.""" + self.__rpc.callback = callback + callback = property(__get_callback, __set_callback) + + @property + def request(self): + """Return the request protocol buffer object.""" + return self.__rpc.request + + @property + def response(self): + """Return the response protocol buffer object.""" + return self.__rpc.response + + @property + def state(self): + """Return the RPC state. + + Possible values are attributes of apiproxy_rpc.RPC: IDLE, RUNNING, + FINISHING. + """ + return self.__rpc.state + + @property + def get_result_hook(self): + """Return the get-result hook function.""" + return self.__get_result_hook + + @property + def user_data(self): + """Return the user data for the hook function.""" + return self.__user_data + + def make_call(self, method, request, response, + get_result_hook=None, user_data=None): + """Initiate a call. + + Args: + method: The method name. + request: The request protocol buffer. + response: The response protocol buffer. + get_result_hook: Optional get-result hook function. If not None, + this must be a function with exactly one argument, the RPC + object (self). Its return value is returned from get_result(). + user_data: Optional additional arbitrary data for the get-result + hook function. This can be accessed as rpc.user_data. The + type of this value is up to the service module. + + This function may only be called once per RPC object. It sends + the request to the remote server, but does not wait for a + response. This allows concurrent execution of the remote call and + further local processing (e.g., making additional remote calls). + + Before the call is initiated, the precall hooks are called. + """ + assert self.__rpc.state == apiproxy_rpc.RPC.IDLE, repr(self.state) + self.__method = method + self.__get_result_hook = get_result_hook + self.__user_data = user_data + apiproxy.GetPreCallHooks().Call( + self.__service, method, request, response, self.__rpc) + self.__rpc.MakeCall(self.__service, method, request, response) + + def wait(self): + """Wait for the call to complete, and call callbacks. + + This is the only time callback functions may be called. (However, + note that check_success() and get_result() call wait().) Waiting + for one RPC may cause callbacks for other RPCs to be called. + Callback functions may call check_success() and get_result(). + + Callbacks are called without arguments; if a callback needs access + to the RPC object a Python nested function (a.k.a. closure) or a + bound may be used. To facilitate this, the callback may be + assigned after the RPC object is created (but before make_call() + is called). + + Note: don't confuse callbacks with get-result hooks or precall + and postcall hooks. + """ + assert self.__rpc.state != apiproxy_rpc.RPC.IDLE, repr(self.state) + if self.__rpc.state == apiproxy_rpc.RPC.RUNNING: + self.__rpc.Wait() + assert self.__rpc.state == apiproxy_rpc.RPC.FINISHING, repr(self.state) + + def check_success(self): + """Check for success of the RPC, possibly raising an exception. + + This function should be called at least once per RPC. If wait() + hasn't been called yet, it is called first. If the RPC caused + an exceptional condition, an exception will be raised here. + The first time check_success() is called, the postcall hooks + are called. + """ + self.wait() + self.__rpc.CheckSuccess() + if not self.__postcall_hooks_called: + self.__postcall_hooks_called = True + apiproxy.GetPostCallHooks().Call(self.__service, self.__method, + self.request, self.response, self.__rpc) + + def get_result(self): + """Get the result of the RPC, or possibly raise an exception. + + This implies a call to check_success(). If a get-result hook was + passed to make_call(), that hook is responsible for calling + check_success(), and the return value of the hook is returned. + Otherwise, check_success() is called directly and None is + returned. + """ + if self.__get_result_hook is None: + self.check_success() + return None + else: + return self.__get_result_hook(self) def GetDefaultAPIProxy(): @@ -249,4 +486,5 @@ def GetDefaultAPIProxy(): except (AttributeError, ImportError): return APIProxyStubMap() + apiproxy = GetDefaultAPIProxy() google_appengine/google/appengine/api/apiproxy_stub_map.py @@ -68,7 +68,9 @@ class AppLogsHandler(logging.StreamHandler): StreamHandler.emit().""" try: message = self._AppLogsMessage(record) - self.stream.write(message.encode("UTF-8")) + if isinstance(message, unicode): + message = message.encode("UTF-8") + self.stream.write(message) self.flush() except (KeyboardInterrupt, SystemExit): raise google_appengine/google/appengine/api/app_logging.py @@ -15,7 +15,7 @@ # limitations under the License. # -"""AppInfo tools +"""AppInfo tools. Library for working with AppInfo records in memory, store and load from configuration files. @@ -29,8 +29,8 @@ import re from google.appengine.api import appinfo_errors from google.appengine.api import validation -from google.appengine.api import yaml_listener from google.appengine.api import yaml_builder +from google.appengine.api import yaml_listener from google.appengine.api import yaml_object @@ -40,11 +40,13 @@ _FILES_REGEX = r'(?!\^).*(?!\$).' _DELTA_REGEX = r'([1-9][0-9]*)([DdHhMm]|[sS]?)' _EXPIRATION_REGEX = r'\s*(%s)(\s+%s)*\s*' % (_DELTA_REGEX, _DELTA_REGEX) +_SERVICE_RE_STRING = r'(mail|xmpp_message)' + _EXPIRATION_CONVERSIONS = { - 'd': 60 * 60 * 24, - 'h': 60 * 60, - 'm': 60, - 's': 1, + 'd': 60 * 60 * 24, + 'h': 60 * 60, + 'm': 60, + 's': 1, } APP_ID_MAX_LEN = 100 @@ -56,6 +58,8 @@ VERSION_RE_STRING = r'(?!-)[a-z\d\-]{1,%d}' % MAJOR_VERSION_ID_MAX_LEN RUNTIME_RE_STRING = r'[a-z]{1,30}' +API_VERSION_RE_STRING = r'[\w.]{1,32}' + HANDLER_STATIC_FILES = 'static_files' HANDLER_STATIC_DIR = 'static_dir' HANDLER_SCRIPT = 'script' @@ -67,18 +71,21 @@ LOGIN_ADMIN = 'admin' SECURE_HTTP = 'never' SECURE_HTTPS = 'always' SECURE_HTTP_OR_HTTPS = 'optional' - -DEFAULT_SKIP_FILES = (r"^(.*/)?(" - r"(app\.yaml)|" - r"(app\.yml)|" - r"(index\.yaml)|" - r"(index\.yml)|" - r"(#.*#)|" - r"(.*~)|" - r"(.*\.py[co])|" - r"(.*/RCS/.*)|" - r"(\..*)|" - r")$") +SECURE_DEFAULT = 'default' + +REQUIRE_MATCHING_FILE = 'require_matching_file' + +DEFAULT_SKIP_FILES = (r'^(.*/)?(' + r'(app\.yaml)|' + r'(app\.yml)|' + r'(index\.yaml)|' + r'(index\.yml)|' + r'(#.*#)|' + r'(.*~)|' + r'(.*\.py[co])|' + r'(.*/RCS/.*)|' + r'(\..*)|' + r')$') LOGIN = 'login' SECURE = 'secure' @@ -97,6 +104,7 @@ API_VERSION = 'api_version' HANDLERS = 'handlers' DEFAULT_EXPIRATION = 'default_expiration' SKIP_FILES = 'skip_files' +SERVICES = 'inbound_services' class URLMap(validation.Validated): @@ -172,39 +180,43 @@ class URLMap(validation.Validated): ATTRIBUTES = { - URL: validation.Optional(_URL_REGEX), - LOGIN: validation.Options(LOGIN_OPTIONAL, - LOGIN_REQUIRED, - LOGIN_ADMIN, - default=LOGIN_OPTIONAL), + URL: validation.Optional(_URL_REGEX), + LOGIN: validation.Options(LOGIN_OPTIONAL, + LOGIN_REQUIRED, + LOGIN_ADMIN, + default=LOGIN_OPTIONAL), + + SECURE: validation.Options(SECURE_HTTP, + SECURE_HTTPS, + SECURE_HTTP_OR_HTTPS, + SECURE_DEFAULT, + default=SECURE_DEFAULT), - SECURE: validation.Options(SECURE_HTTP, - SECURE_HTTPS, - SECURE_HTTP_OR_HTTPS, - default=SECURE_HTTP), + HANDLER_STATIC_FILES: validation.Optional(_FILES_REGEX), + UPLOAD: validation.Optional(_FILES_REGEX), - HANDLER_STATIC_FILES: validation.Optional(_FILES_REGEX), - UPLOAD: validation.Optional(_FILES_REGEX), + HANDLER_STATIC_DIR: validation.Optional(_FILES_REGEX), - HANDLER_STATIC_DIR: validation.Optional(_FILES_REGEX), + MIME_TYPE: validation.Optional(str), + EXPIRATION: validation.Optional(_EXPIRATION_REGEX), - MIME_TYPE: validation.Optional(str), - EXPIRATION: validation.Optional(_EXPIRATION_REGEX), + HANDLER_SCRIPT: validation.Optional(_FILES_REGEX), - HANDLER_SCRIPT: validation.Optional(_FILES_REGEX), + REQUIRE_MATCHING_FILE: validation.Optional(bool), } COMMON_FIELDS = set([URL, LOGIN, SECURE]) ALLOWED_FIELDS = { - HANDLER_STATIC_FILES: (MIME_TYPE, UPLOAD, EXPIRATION), - HANDLER_STATIC_DIR: (MIME_TYPE, EXPIRATION), - HANDLER_SCRIPT: (), + HANDLER_STATIC_FILES: (MIME_TYPE, UPLOAD, EXPIRATION, + REQUIRE_MATCHING_FILE), + HANDLER_STATIC_DIR: (MIME_TYPE, EXPIRATION, REQUIRE_MATCHING_FILE), + HANDLER_SCRIPT: (), } def GetHandler(self): @@ -246,9 +258,9 @@ class URLMap(validation.Validated): not (attribute in allowed_fields or attribute in URLMap.COMMON_FIELDS or attribute == mapping_type)): - raise appinfo_errors.UnexpectedHandlerAttribute( - 'Unexpected attribute "%s" for mapping type %s.' % - (attribute, mapping_type)) + raise appinfo_errors.UnexpectedHandlerAttribute( + 'Unexpected attribute "%s" for mapping type %s.' % + (attribute, mapping_type)) if mapping_type == HANDLER_STATIC_FILES and not self.upload: raise appinfo_errors.MissingHandlerAttribute( @@ -302,15 +314,18 @@ class AppInfoExternal(validation.Validated): ATTRIBUTES = { - APPLICATION: APPLICATION_RE_STRING, - VERSION: VERSION_RE_STRING, - RUNTIME: RUNTIME_RE_STRING, + APPLICATION: APPLICATION_RE_STRING, + VERSION: VERSION_RE_STRING, + RUNTIME: RUNTIME_RE_STRING, - API_VERSION: validation.Options('1', 'beta'), - HANDLERS: validation.Optional(validation.Repeated(URLMap)), - DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), - SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES) + API_VERSION: API_VERSION_RE_STRING, + HANDLERS: validation.Optional(validation.Repeated(URLMap)), + + SERVICES: validation.Optional(validation.Repeated( + validation.Regex(_SERVICE_RE_STRING))), + DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), + SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES) } def CheckInitialized(self): @@ -329,6 +344,19 @@ class AppInfoExternal(validation.Validated): 'Found more than %d URLMap entries in application configuration' % MAX_URL_MAPS) + def FixSecureDefaults(self): + """Force omitted 'secure: ...' handler fields to 'secure: optional'. + + The effect is that handler.secure is never equal to the (nominal) + default. + + See http://b/issue?id=2073962. + """ + if self.handlers: + for handler in self.handlers: + if handler.secure == SECURE_DEFAULT: + handler.secure = SECURE_HTTP_OR_HTTPS + def LoadSingleAppInfo(app_info): """Load a single AppInfo object where one and only one is expected. @@ -342,8 +370,9 @@ def LoadSingleAppInfo(app_info): An instance of AppInfoExternal as loaded from a YAML file. Raises: - EmptyConfigurationFile when there are no documents in YAML file. - MultipleConfigurationFile when there is more than one document in YAML + ValueError: if a specified service is not valid. + EmptyConfigurationFile: when there are no documents in YAML file. + MultipleConfigurationFile: when there is more than one document in YAML file. """ builder = yaml_object.ObjectBuilder(AppInfoExternal) @@ -356,6 +385,7 @@ def LoadSingleAppInfo(app_info): raise appinfo_errors.EmptyConfigurationFile() if len(app_infos) > 1: raise appinfo_errors.MultipleConfigurationFile() + app_infos[0].FixSecureDefaults() return app_infos[0] @@ -379,7 +409,7 @@ def ParseExpiration(expiration): _file_path_positive_re = re.compile(r'^[ 0-9a-zA-Z\._\+/\$-]{1,256}$') -_file_path_negative_1_re = re.compile(r'\.\.|^\./|\.$|/\./|^-') +_file_path_negative_1_re = re.compile(r'\.\.|^\./|\.$|/\./|^-|^_ah/') _file_path_negative_2_re = re.compile(r'//|/$') @@ -406,7 +436,8 @@ def ValidFilename(filename): if _file_path_positive_re.match(filename) is None: return 'Invalid character in filename: %s' % filename if _file_path_negative_1_re.search(filename) is not None: - return ('Filename cannot contain "." or ".." or start with "-": %s' % + return ('Filename cannot contain "." or ".." ' + 'or start with "-" or "_ah/": %s' % filename) if _file_path_negative_2_re.search(filename) is not None: return 'Filename cannot have trailing / or contain //: %s' % filename google_appengine/google/appengine/api/appinfo.py @@ -39,8 +39,9 @@ class IsEnabledRequest(ProtocolBuffer.ProtocolMessage): self.package_ = x def clear_package(self): - self.has_package_ = 0 - self.package_ = "" + if self.has_package_: + self.has_package_ = 0 + self.package_ = "" def has_package(self): return self.has_package_ @@ -158,26 +159,27 @@ class IsEnabledRequest(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kpackage = 1 kcapability = 2 kcall = 3 - _TEXT = ( - "ErrorCode", - "package", - "capability", - "call", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "package", + 2: "capability", + 3: "call", + }, 3) - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + }, 3, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -216,8 +218,9 @@ class IsEnabledResponse(ProtocolBuffer.ProtocolMessage): self.summary_status_ = x def clear_summary_status(self): - self.has_summary_status_ = 0 - self.summary_status_ = 0 + if self.has_summary_status_: + self.has_summary_status_ = 0 + self.summary_status_ = 0 def has_summary_status(self): return self.has_summary_status_ @@ -228,8 +231,9 @@ class IsEnabledResponse(ProtocolBuffer.ProtocolMessage): self.time_until_scheduled_ = x def clear_time_until_scheduled(self): - self.has_time_until_scheduled_ = 0 - self.time_until_scheduled_ = 0 + if self.has_time_until_scheduled_: + self.has_time_until_scheduled_ = 0 + self.time_until_scheduled_ = 0 def has_time_until_scheduled(self): return self.has_time_until_scheduled_ @@ -334,26 +338,27 @@ class IsEnabledResponse(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + ksummary_status = 1 ktime_until_scheduled = 2 kconfig = 3 - _TEXT = ( - "ErrorCode", - "summary_status", - "time_until_scheduled", - "config", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "summary_status", + 2: "time_until_scheduled", + 3: "config", + }, 3) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.STRING, + }, 3, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" google_appengine/google/appengine/api/capabilities/capability_service_pb.py @@ -23,6 +23,15 @@ application. Supports loading the records from yaml. +import logging +import sys +import traceback + +try: + import pytz +except ImportError: + pytz = None + from google.appengine.cron import groc from google.appengine.api import validation from google.appengine.api import yaml_builder @@ -30,10 +39,7 @@ from google.appengine.api import yaml_listener from google.appengine.api import yaml_object _URL_REGEX = r'^/.*$' - - _TIMEZONE_REGEX = r'^.{0,100}$' - _DESCRIPTION_REGEX = r'^.{0,499}$' @@ -55,6 +61,31 @@ class GrocValidator(validation.Validator): return value +class TimezoneValidator(validation.Validator): + """Checks that a timezone can be correctly parsed and is known.""" + + def Validate(self, value): + """Validates a timezone.""" + if value is None: + return + if not isinstance(value, basestring): + raise TypeError('timezone must be a string, not \'%r\'' % type(value)) + if pytz is None: + return value + try: + pytz.timezone(value) + except pytz.UnknownTimeZoneError: + raise validation.ValidationError('timezone \'%s\' is unknown' % value) + except IOError: + return value + except: + unused_e, v, t = sys.exc_info() + logging.warning('pytz raised an unexpected error: %s.\n' % (v) + + 'Traceback:\n' + '\n'.join(traceback.format_tb(t))) + raise + return value + + CRON = 'cron' URL = 'url' @@ -73,7 +104,7 @@ class CronEntry(validation.Validated): ATTRIBUTES = { URL: _URL_REGEX, SCHEDULE: GrocValidator(), - TIMEZONE: validation.Optional(_TIMEZONE_REGEX), + TIMEZONE: TimezoneValidator(), DESCRIPTION: validation.Optional(_DESCRIPTION_REGEX) } google_appengine/google/appengine/api/croninfo.py @@ -31,6 +31,8 @@ which needs BadValueError, so it can't be defined in datastore. +import heapq +import itertools import logging import re import string @@ -40,6 +42,7 @@ from xml.sax import saxutils from google.appengine.api import api_base_pb from google.appengine.api import apiproxy_stub_map +from google.appengine.api import capabilities from google.appengine.api import datastore_errors from google.appengine.api import datastore_types from google.appengine.datastore import datastore_index @@ -47,10 +50,28 @@ from google.appengine.datastore import datastore_pb from google.appengine.runtime import apiproxy_errors from google.appengine.datastore import entity_pb -TRANSACTION_RETRIES = 3 +try: + __import__('google.appengine.api.labs.taskqueue.taskqueue_service_pb') + taskqueue_service_pb = sys.modules.get( + 'google.appengine.api.labs.taskqueue.taskqueue_service_pb') +except ImportError: + from google.appengine.api.taskqueue import taskqueue_service_pb + +MAX_ALLOWABLE_QUERIES = 30 + +MAXIMUM_RESULTS = 1000 + +DEFAULT_TRANSACTION_RETRIES = 3 + +READ_CAPABILITY = capabilities.CapabilitySet('datastore_v3') +WRITE_CAPABILITY = capabilities.CapabilitySet( + 'datastore_v3', + capabilities=['write']) _MAX_INDEXED_PROPERTIES = 5000 +_MAX_ID_BATCH_SIZE = 1000 * 1000 * 1000 + Key = datastore_types.Key typename = datastore_types.typename @@ -143,7 +164,7 @@ def Put(entities): return [] for entity in entities: - if not entity.kind() or not entity.app(): + if not entity.kind() or not entity.app_id_namespace(): raise datastore_errors.BadRequestError( 'App and kind must not be empty, in entity: %s' % entity) @@ -152,8 +173,6 @@ def Put(entities): keys = [e.key() for e in entities] tx = _MaybeSetupTransaction(req, keys) - if tx: - tx.RecordModifiedKeys([k for k in keys if k.has_id_or_name()]) resp = datastore_pb.PutResponse() try: @@ -173,7 +192,7 @@ def Put(entities): entity._Entity__key._Key__reference.CopyFrom(key) if tx: - tx.RecordModifiedKeys([e.key() for e in entities], error_on_repeat=False) + tx.entity_group = entities[0].entity_group() if multiple: return [Key._FromPb(k) for k in keys] @@ -254,8 +273,6 @@ def Delete(keys): req.key_list().extend([key._Key__reference for key in keys]) tx = _MaybeSetupTransaction(req, keys) - if tx: - tx.RecordModifiedKeys(keys) resp = datastore_pb.DeleteResponse() try: @@ -270,7 +287,8 @@ class Entity(dict): Includes read-only accessors for app id, kind, and primary key. Also provides dictionary-style access to properties. """ - def __init__(self, kind, parent=None, _app=None, name=None): + def __init__(self, kind, parent=None, _app=None, name=None, id=None, + unindexed_properties=[], _namespace=None): """Constructor. Takes the kind and transaction root, which cannot be changed after the entity is constructed, and an optional parent. Raises BadArgumentError or BadKeyError if kind is invalid or parent is not an @@ -283,43 +301,80 @@ class Entity(dict): parent: Entity or Key # if provided, this entity's name. name: string + # if provided, this entity's id. + id: integer + # if provided, a sequence of property names that should not be indexed + # by the built-in single property indices. + unindexed_properties: list or tuple of strings """ ref = entity_pb.Reference() - _app = datastore_types.ResolveAppId(_app) - ref.set_app(_app) + _app_namespace = datastore_types.ResolveAppIdNamespace(_app, _namespace) + ref.set_app(_app_namespace.to_encoded()) datastore_types.ValidateString(kind, 'kind', datastore_errors.BadArgumentError) - if parent is not None: parent = _GetCompleteKeyOrError(parent) - if _app != parent.app(): + if _app_namespace != parent.app_id_namespace(): raise datastore_errors.BadArgumentError( - "_app %s doesn't match parent's app %s" % (_app, parent.app())) + " %s doesn't match parent's app_namespace %s" % + (_app_namespace, parent.app_id_namespace())) ref.CopyFrom(parent._Key__reference) last_path = ref.mutable_path().add_element() last_path.set_type(kind.encode('utf-8')) + if name is not None and id is not None: + raise datastore_errors.BadArgumentError( + "Cannot set both name and id on an Entity") + if name is not None: datastore_types.ValidateString(name, 'name') - if name[0] in string.digits: - raise datastore_errors.BadValueError('name cannot begin with a digit') last_path.set_name(name.encode('utf-8')) + if id is not None: + datastore_types.ValidateInteger(id, 'id') + last_path.set_id(id) + + unindexed_properties, multiple = NormalizeAndTypeCheck(unindexed_properties, basestring) + if not multiple: + raise datastore_errors.BadArgumentError( + 'unindexed_properties must be a sequence; received %s (a %s).' % + (unindexed_properties, typename(unindexed_properties))) + for prop in unindexed_properties: + datastore_types.ValidateProperty(prop, None) + self.__unindexed_properties = frozenset(unindexed_properties) + self.__key = Key._FromPb(ref) def app(self): """Returns the name of the application that created this entity, a - string. + string or None if not set. """ return self.__key.app() + def namespace(self): + """Returns the namespace of this entity, a string or None. + """ + return self.__key.namespace() + + def app_id_namespace(self): + """Returns the AppIdNamespace of this entity or None if not set. + """ + return self.__key.app_id_namespace() + def kind(self): """Returns this entity's kind, a string. """ return self.__key.kind() + def is_saved(self): + """Returns if this entity has been saved to the datastore + """ + last_path = self.__key._Key__reference.path().element_list()[-1] + return ((last_path.has_name() ^ last_path.has_id()) and + self.__key.has_id_or_name()) + def key(self): """Returns this entity's primary key, a Key instance. """ @@ -332,13 +387,17 @@ class Entity(dict): return self.key().parent() def entity_group(self): - """Returns this entitys's entity group as a Key. + """Returns this entity's entity group as a Key. Note that the returned Key will be incomplete if this is a a root entity and its key is incomplete. """ return self.key().entity_group() + def unindexed_properties(self): + """Returns this entity's unindexed properties, as a frozenset of strings.""" + return getattr(self, '_Entity__unindexed_properties', []) + def __setitem__(self, name, value): """Implements the [] operator. Used to set property value(s). @@ -461,7 +520,15 @@ class Entity(dict): return xml - def _ToPb(self): + def ToPb(self): + """Converts this Entity to its protocol buffer representation. + + Returns: + entity_pb.Entity + """ + return self._ToPb(False) + + def _ToPb(self, mark_key_as_saved=True): """Converts this Entity to its protocol buffer representation. Not intended to be used by application developers. @@ -471,6 +538,9 @@ class Entity(dict): pb = entity_pb.EntityProto() pb.mutable_key().CopyFrom(self.key()._ToPb()) + last_path = pb.key().path().element_list()[-1] + if mark_key_as_saved and last_path.has_name() and last_path.has_id(): + last_path.clear_id() group = pb.mutable_entity_group() if self.__key.has_id_or_name(): @@ -488,7 +558,8 @@ class Entity(dict): if isinstance(sample, list): sample = values[0] - if isinstance(sample, (datastore_types.Blob, datastore_types.Text)): + if (isinstance(sample, datastore_types._RAW_PROPERTY_TYPES) or + name in self.unindexed_properties()): pb.raw_property_list().extend(properties) else: pb.property_list().extend(properties) @@ -500,7 +571,25 @@ class Entity(dict): return pb @staticmethod - def _FromPb(pb): + def FromPb(pb): + """Static factory method. Returns the Entity representation of the + given protocol buffer (datastore_pb.Entity). + + Args: + pb: datastore_pb.Entity or str encoding of a datastore_pb.Entity + + Returns: + Entity: the Entity representation of pb + """ + if isinstance(pb, str): + real_pb = entity_pb.EntityProto() + real_pb.ParseFromString(pb) + pb = real_pb + + return Entity._FromPb(pb, require_valid_key=False) + + @staticmethod + def _FromPb(pb, require_valid_key=True): """Static factory method. Returns the Entity representation of the given protocol buffer (datastore_pb.Entity). Not intended to be used by application developers. @@ -519,14 +608,19 @@ class Entity(dict): assert pb.key().path().element_size() > 0 last_path = pb.key().path().element_list()[-1] - assert last_path.has_id() ^ last_path.has_name() - if last_path.has_id(): - assert last_path.id() != 0 - else: - assert last_path.has_name() - assert last_path.name() + if require_valid_key: + assert last_path.has_id() ^ last_path.has_name() + if last_path.has_id(): + assert last_path.id() != 0 + else: + assert last_path.has_name() + assert last_path.name() + + unindexed_properties = [p.name() for p in pb.raw_property_list()] - e = Entity(unicode(last_path.type().decode('utf-8'))) + e = Entity(unicode(last_path.type().decode('utf-8')), + unindexed_properties=unindexed_properties, + _app=pb.key().app()) ref = e.__key._Key__reference ref.CopyFrom(pb.key()) @@ -534,11 +628,6 @@ class Entity(dict): for prop_list in (pb.property_list(), pb.raw_property_list()): for prop in prop_list: - if not prop.has_multiple(): - raise datastore_errors.Error( - 'Property %s is corrupt in the datastore; it\'s missing the ' - 'multiple valued field.' % prop.name()) - try: value = datastore_types.FromPropertyPb(prop) except (AssertionError, AttributeError, TypeError, ValueError), e: @@ -673,6 +762,9 @@ class Query(dict): __cached_count = None __hint = None __ancestor = None + __compile = None + + __cursor = None __filter_order = None __filter_counter = 0 @@ -680,7 +772,8 @@ class Query(dict): __inequality_prop = None __inequality_count = 0 - def __init__(self, kind, filters={}, _app=None): + def __init__(self, kind=None, filters={}, _app=None, keys_only=False, + compile=True, cursor=None, _namespace=None): """Constructor. Raises BadArgumentError if kind is not a string. Raises BadValueError or @@ -688,19 +781,25 @@ class Query(dict): Args: # kind is required. filters is optional; if provided, it's used - # as an initial set of property filters. + # as an initial set of property filters. keys_only defaults to False. kind: string filters: dict + keys_only: boolean """ - datastore_types.ValidateString(kind, 'kind', - datastore_errors.BadArgumentError) + if kind is not None: + datastore_types.ValidateString(kind, 'kind', + datastore_errors.BadArgumentError) self.__kind = kind self.__orderings = [] self.__filter_order = {} self.update(filters) - self.__app = datastore_types.ResolveAppId(_app) + self.__app = datastore_types.ResolveAppIdNamespace(_app, + _namespace).to_encoded() + self.__keys_only = keys_only + self.__compile = compile + self.__cursor = cursor def Order(self, *orderings): """Specify how the query results should be sorted. @@ -771,6 +870,13 @@ class Query(dict): str(direction)) direction = Query.ASCENDING + if (self.__kind is None and + (property != datastore_types._KEY_SPECIAL_PROPERTY or + direction != Query.ASCENDING)): + raise datastore_errors.BadArgumentError( + 'Only %s ascending orders are supported on kindless queries' % + datastore_types._KEY_SPECIAL_PROPERTY) + orderings[i] = (property, direction) if (orderings and self.__inequality_prop and @@ -838,11 +944,21 @@ class Query(dict): # this query Query """ - key = _GetCompleteKeyOrError(ancestor) - self.__ancestor = datastore_pb.Reference() - self.__ancestor.CopyFrom(key._Key__reference) + self.__ancestor = _GetCompleteKeyOrError(ancestor) return self + def IsKeysOnly(self): + """Returns True if this query is keys only, false otherwise.""" + return self.__keys_only + + def GetCompiledQuery(self): + try: + return self.__compiled_query + except AttributeError: + raise AssertionError('No cursor available, either this query has not ' + 'been executed or there is no compilation ' + 'available for this kind of query') + def Run(self): """Runs this query. @@ -857,36 +973,44 @@ class Query(dict): # an iterator that provides access to the query results Iterator """ + self.__compile = False return self._Run() - def _Run(self, limit=None, offset=None): + def _Run(self, limit=None, offset=None, + prefetch_count=None, next_count=None): """Runs this query, with an optional result limit and an optional offset. - Identical to Run, with the extra optional limit and offset parameters. - limit and offset must both be integers >= 0. + Identical to Run, with the extra optional limit, offset, prefetch_count, + next_count parameters. These parameters must be integers >= 0. This is not intended to be used by application developers. Use Get() instead! """ - if _CurrentTransactionKey(): - raise datastore_errors.BadRequestError( - "Can't query inside a transaction.") - - pb = self._ToPb(limit, offset) + pb = self._ToPb(limit, offset, prefetch_count) result = datastore_pb.QueryResult() + api_call = 'RunQuery' + if self.__cursor: + pb = self._ToCompiledPb(pb, self.__cursor, prefetch_count) + api_call = 'RunCompiledQuery' try: - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'RunQuery', pb, result) + apiproxy_stub_map.MakeSyncCall('datastore_v3', api_call, pb, result) except apiproxy_errors.ApplicationError, err: try: - _ToDatastoreError(err) + raise _ToDatastoreError(err) except datastore_errors.NeedIndexError, exc: yaml = datastore_index.IndexYamlForQuery( *datastore_index.CompositeIndexForQuery(pb)[1:-1]) raise datastore_errors.NeedIndexError( str(exc) + '\nThis query needs this index:\n' + yaml) - return Iterator._FromPb(result.cursor()) + if self.__compile: + if result.has_compiled_query(): + self.__compiled_query = result.compiled_query() + else: + self.__compiled_query = None + + return Iterator(result, batch_size=next_count) def Get(self, limit, offset=0): """Fetches and returns a maximum number of results from the query. @@ -935,7 +1059,8 @@ class Query(dict): 'Argument to Get named \'offset\' must be an int greater than or ' 'equal to 0; received %s (a %s)' % (offset, typename(offset))) - return self._Run(limit, offset)._Next(limit) + return self._Run(limit=limit, offset=offset, + prefetch_count=limit)._Get(limit) def Count(self, limit=None): """Returns the number of entities that this query matches. The returned @@ -949,6 +1074,7 @@ class Query(dict): Returns: The number of results. """ + self.__compile = False if self.__cached_count: return self.__cached_count @@ -1072,12 +1198,9 @@ class Query(dict): values = list(values) elif not isinstance(values, list): values = [values] - if isinstance(values[0], datastore_types.Blob): + if isinstance(values[0], datastore_types._RAW_PROPERTY_TYPES): raise datastore_errors.BadValueError( - 'Filtering on Blob properties is not supported.') - if isinstance(values[0], datastore_types.Text): - raise datastore_errors.BadValueError( - 'Filtering on Text properties is not supported.') + 'Filtering on %s properties is not supported.' % typename(values[0])) if operator in self.INEQUALITY_OPERATORS: if self.__inequality_prop and property != self.__inequality_prop: @@ -1090,6 +1213,12 @@ class Query(dict): 'first sort order, if any sort orders are supplied' % ', '.join(self.INEQUALITY_OPERATORS)) + if (self.__kind is None and + property != datastore_types._KEY_SPECIAL_PROPERTY): + raise datastore_errors.BadFilterError( + 'Only %s filters are allowed on kindless queries.' % + datastore_types._KEY_SPECIAL_PROPERTY) + if property in datastore_types._SPECIAL_PROPERTIES: if property == datastore_types._KEY_SPECIAL_PROPERTY: for value in values: @@ -1100,7 +1229,7 @@ class Query(dict): return match - def _ToPb(self, limit=None, offset=None): + def _ToPb(self, limit=None, offset=None, count=None): """Converts this Query to its protocol buffer representation. Not intended to be used by application developers. Enforced by hiding the datastore_pb classes. @@ -1111,22 +1240,40 @@ class Query(dict): # number of results that match the query to skip. limit is applied # after the offset is fulfilled offset: int + # the requested initial batch size + count: int Returns: # the PB representation of this Query datastore_pb.Query + + Raises: + BadRequestError if called inside a transaction and the query does not + include an ancestor. """ + + if not self.__ancestor and _CurrentTransactionKey(): + raise datastore_errors.BadRequestError( + 'Only ancestor queries are allowed inside transactions.') + pb = datastore_pb.Query() + _MaybeSetupTransaction(pb, [self.__ancestor]) - pb.set_kind(self.__kind.encode('utf-8')) + if self.__kind is not None: + pb.set_kind(self.__kind.encode('utf-8')) + pb.set_keys_only(bool(self.__keys_only)) if self.__app: pb.set_app(self.__app.encode('utf-8')) + if self.__compile: + pb.set_compile(True) if limit is not None: pb.set_limit(limit) if offset is not None: pb.set_offset(offset) + if count is not None: + pb.set_count(count) if self.__ancestor: - pb.mutable_ancestor().CopyFrom(self.__ancestor) + pb.mutable_ancestor().CopyFrom(self.__ancestor._Key__reference) if ((self.__hint == self.ORDER_FIRST and self.__orderings) or (self.__hint == self.ANCESTOR_FIRST and self.__ancestor) or @@ -1164,6 +1311,371 @@ class Query(dict): return pb + def _ToCompiledPb(self, query_pb, cursor, count=None): + compiled_pb = datastore_pb.RunCompiledQueryRequest() + compiled_pb.mutable_original_query().CopyFrom(query_pb) + compiled_pb.mutable_compiled_query().CopyFrom(cursor) + if count is not None: + compiled_pb.set_count(count) + return compiled_pb + +def AllocateIds(model_key, size): + """Allocates a range of IDs of size for the key defined by model_key + + Allocates a range of IDs in the datastore such that those IDs will not + be automatically assigned to new entities. You can only allocate IDs + for model keys from your app. If there is an error, raises a subclass of + datastore_errors.Error. + + Args: + model_key: Key or string to serve as a model specifying the ID sequence + in which to allocate IDs + + Returns: + (start, end) of the allocated range, inclusive. + """ + keys, multiple = NormalizeAndTypeCheckKeys(model_key) + + if len(keys) > 1: + raise datastore_errors.BadArgumentError( + 'Cannot allocate IDs for more than one model key at a time') + + if size > _MAX_ID_BATCH_SIZE: + raise datastore_errors.BadArgumentError( + 'Cannot allocate more than %s ids at a time' % _MAX_ID_BATCH_SIZE) + if size <= 0: + raise datastore_errors.BadArgumentError( + 'Cannot allocate less than 1 id') + + req = datastore_pb.AllocateIdsRequest() + req.mutable_model_key().CopyFrom(keys[0]._ToPb()) + req.set_size(size) + + resp = datastore_pb.AllocateIdsResponse() + try: + apiproxy_stub_map.MakeSyncCall('datastore_v3', 'AllocateIds', req, resp) + except apiproxy_errors.ApplicationError, err: + raise _ToDatastoreError(err) + + return resp.start(), resp.end() + + +class MultiQuery(Query): + """Class representing a query which requires multiple datastore queries. + + This class is actually a subclass of datastore.Query as it is intended to act + like a normal Query object (supporting the same interface). + + Does not support keys only queries, since it needs whole entities in order + to merge sort them. (That's not true if there are no sort orders, or if the + sort order is on __key__, but allowing keys only queries in those cases, but + not in others, would be confusing.) + """ + + def __init__(self, bound_queries, orderings): + if len(bound_queries) > MAX_ALLOWABLE_QUERIES: + raise datastore_errors.BadArgumentError( + 'Cannot satisfy query -- too many subqueries (max: %d, got %d).' + ' Probable cause: too many IN/!= filters in query.' % + (MAX_ALLOWABLE_QUERIES, len(bound_queries))) + + for query in bound_queries: + if query.IsKeysOnly(): + raise datastore_errors.BadQueryError( + 'MultiQuery does not support keys_only.') + + self.__bound_queries = bound_queries + self.__orderings = orderings + self.__compile = False + + def __str__(self): + res = 'MultiQuery: ' + for query in self.__bound_queries: + res = '%s %s' % (res, str(query)) + return res + + def Get(self, limit, offset=0): + """Get results of the query with a limit on the number of results. + + Args: + limit: maximum number of values to return. + offset: offset requested -- if nonzero, this will override the offset in + the original query + + Returns: + A list of entities with at most "limit" entries (less if the query + completes before reading limit values). + """ + count = 1 + result = [] + + iterator = self.Run() + + try: + for i in xrange(offset): + val = iterator.next() + except StopIteration: + pass + + try: + while count <= limit: + val = iterator.next() + result.append(val) + count += 1 + except StopIteration: + pass + return result + + class SortOrderEntity(object): + """Allow entity comparisons using provided orderings. + + The iterator passed to the constructor is eventually consumed via + calls to GetNext(), which generate new SortOrderEntity s with the + same orderings. + """ + + def __init__(self, entity_iterator, orderings): + """Ctor. + + Args: + entity_iterator: an iterator of entities which will be wrapped. + orderings: an iterable of (identifier, order) pairs. order + should be either Query.ASCENDING or Query.DESCENDING. + """ + self.__entity_iterator = entity_iterator + self.__entity = None + self.__min_max_value_cache = {} + try: + self.__entity = entity_iterator.next() + except StopIteration: + pass + else: + self.__orderings = orderings + + def __str__(self): + return str(self.__entity) + + def GetEntity(self): + """Gets the wrapped entity.""" + return self.__entity + + def GetNext(self): + """Wrap and return the next entity. + + The entity is retrieved from the iterator given at construction time. + """ + return MultiQuery.SortOrderEntity(self.__entity_iterator, + self.__orderings) + + def CmpProperties(self, that): + """Compare two entities and return their relative order. + + Compares self to that based on the current sort orderings and the + key orders between them. Returns negative, 0, or positive depending on + whether self is less, equal to, or greater than that. This + comparison returns as if all values were to be placed in ascending order + (highest value last). Only uses the sort orderings to compare (ignores + keys). + + Args: + that: SortOrderEntity + + Returns: + Negative if self < that + Zero if self == that + Positive if self > that + """ + if not self.__entity: + return cmp(self.__entity, that.__entity) + + for (identifier, order) in self.__orderings: + value1 = self.__GetValueForId(self, identifier, order) + value2 = self.__GetValueForId(that, identifier, order) + + result = cmp(value1, value2) + if order == Query.DESCENDING: + result = -result + if result: + return result + return 0 + + def __GetValueForId(self, sort_order_entity, identifier, sort_order): + value = _GetPropertyValue(sort_order_entity.__entity, identifier) + if isinstance(value, list): + entity_key = sort_order_entity.__entity.key() + if (entity_key, identifier) in self.__min_max_value_cache: + value = self.__min_max_value_cache[(entity_key, identifier)] + elif sort_order == Query.DESCENDING: + value = min(value) + else: + value = max(value) + self.__min_max_value_cache[(entity_key, identifier)] = value + + return value + + def __cmp__(self, that): + """Compare self to that w.r.t. values defined in the sort order. + + Compare an entity with another, using sort-order first, then the key + order to break ties. This can be used in a heap to have faster min-value + lookup. + + Args: + that: other entity to compare to + Returns: + negative: if self is less than that in sort order + zero: if self is equal to that in sort order + positive: if self is greater than that in sort order + """ + property_compare = self.CmpProperties(that) + if property_compare: + return property_compare + else: + return cmp(self.__entity.key(), that.__entity.key()) + + def Run(self): + """Return an iterable output with all results in order.""" + results = [] + count = 1 + log_level = logging.DEBUG - 1 + for bound_query in self.__bound_queries: + logging.log(log_level, 'Running query #%i' % count) + results.append(bound_query.Run()) + count += 1 + + def IterateResults(results): + """Iterator function to return all results in sorted order. + + Iterate over the array of results, yielding the next element, in + sorted order. This function is destructive (results will be empty + when the operation is complete). + + Args: + results: list of result iterators to merge and iterate through + + Yields: + The next result in sorted order. + """ + result_heap = [] + for result in results: + heap_value = MultiQuery.SortOrderEntity(result, self.__orderings) + if heap_value.GetEntity(): + heapq.heappush(result_heap, heap_value) + + used_keys = set() + + while result_heap: + top_result = heapq.heappop(result_heap) + + results_to_push = [] + if top_result.GetEntity().key() not in used_keys: + yield top_result.GetEntity() + else: + pass + + used_keys.add(top_result.GetEntity().key()) + + results_to_push = [] + while result_heap: + next = heapq.heappop(result_heap) + if cmp(top_result, next): + results_to_push.append(next) + break + else: + results_to_push.append(next.GetNext()) + results_to_push.append(top_result.GetNext()) + + for popped_result in results_to_push: + if popped_result.GetEntity(): + heapq.heappush(result_heap, popped_result) + + return IterateResults(results) + + def Count(self, limit=None): + """Return the number of matched entities for this query. + + Will return the de-duplicated count of results. Will call the more + efficient Get() function if a limit is given. + + Args: + limit: maximum number of entries to count (for any result > limit, return + limit). + Returns: + count of the number of entries returned. + """ + if limit is None: + count = 0 + for i in self.Run(): + count += 1 + return count + else: + return len(self.Get(limit)) + + def GetCompiledQuery(self): + raise AssertionError('No cursor available for a MultiQuery (queries ' + 'using "IN" or "!=" operators)') + + def __setitem__(self, query_filter, value): + """Add a new filter by setting it on all subqueries. + + If any of the setting operations raise an exception, the ones + that succeeded are undone and the exception is propagated + upward. + + Args: + query_filter: a string of the form "property operand". + value: the value that the given property is compared against. + """ + saved_items = [] + for index, query in enumerate(self.__bound_queries): + saved_items.append(query.get(query_filter, None)) + try: + query[query_filter] = value + except: + for q, old_value in itertools.izip(self.__bound_queries[:index], + saved_items): + if old_value is not None: + q[query_filter] = old_value + else: + del q[query_filter] + raise + + def __delitem__(self, query_filter): + """Delete a filter by deleting it from all subqueries. + + If a KeyError is raised during the attempt, it is ignored, unless + every subquery raised a KeyError. If any other exception is + raised, any deletes will be rolled back. + + Args: + query_filter: the filter to delete. + + Raises: + KeyError: No subquery had an entry containing query_filter. + """ + subquery_count = len(self.__bound_queries) + keyerror_count = 0 + saved_items = [] + for index, query in enumerate(self.__bound_queries): + try: + saved_items.append(query.get(query_filter, None)) + del query[query_filter] + except KeyError: + keyerror_count += 1 + except: + for q, old_value in itertools.izip(self.__bound_queries[:index], + saved_items): + if old_value is not None: + q[query_filter] = old_value + raise + + if keyerror_count == subquery_count: + raise KeyError(query_filter) + + def __iter__(self): + return iter(self.__bound_queries) + + class Iterator(object): """An iterator over the results of a datastore query. @@ -1178,74 +1690,146 @@ class Iterator(object): > for person in it: > print 'Hi, %s!' % person['name'] """ - def __init__(self, cursor): - self.__cursor = cursor - self.__buffer = [] - self.__more_results = True + def __init__(self, query_result_pb, batch_size=None): + self.__cursor = query_result_pb.cursor() + self.__keys_only = query_result_pb.keys_only() + self.__batch_size = batch_size + self.__buffer = self._ProcessQueryResult(query_result_pb) - def _Next(self, count): - """Returns the next result(s) of the query. + def _Get(self, count): + """Gets the next count result(s) of the query. Not intended to be used by application developers. Use the python iterator protocol instead. - This method returns the next entities from the list of resulting - entities that matched the query. If the query specified a sort - order, entities are returned in that order. Otherwise, the order - is undefined. - - The argument specifies the number of entities to return. If it's - greater than the number of remaining entities, all of the - remaining entities are returned. In that case, the length of the - returned list will be smaller than count. + This method uses _Next to returns the next entities or keys from the list of + matching results. If the query specified a sort order, results are returned + in that order. Otherwise, the order is undefined. - There is an internal buffer for use with the next() method. If - this buffer is not empty, up to 'count' values are removed from - this buffer and returned. It's best not to mix _Next() and - next(). + The argument, count, specifies the number of results to return. However, the + length of the returned list may be smaller than count. This is the case only + if count is greater than the number of remaining results. - The results are always returned as a list. If there are no results - left, an empty list is returned. + The results are always returned as a list. If there are no results left, + an empty list is returned. Args: - # the number of entities to return; must be >= 1 + # the number of results to return; must be >= 1 count: int or long Returns: - # a list of entities - [Entity, ...] + # a list of entities or keys + [Entity or Key, ...] """ - if not isinstance(count, (int, long)) or count <= 0: + if count > MAXIMUM_RESULTS: + count = MAXIMUM_RESULTS + entity_list = self._Next(count) + while len(entity_list) < count and self.__more_results: + next_results = self._Next(count - len(entity_list)) + if not next_results: + break + entity_list += next_results + return entity_list; + + def _Next(self, count=None): + """Returns the next batch of results. + + Not intended to be used by application developers. Use the python + iterator protocol instead. + + This method returns the next entities or keys from the list of matching + results. If the query specified a sort order, results are returned in that + order. Otherwise, the order is undefined. + + The optional argument, count, specifies the number of results to return. + However, the length of the returned list may be smaller than count. This is + the case if count is greater than the number of remaining results or the + size of the remaining results exceeds the RPC buffer limit. Use _Get to + insure all possible entities are retrieved. + + If the count is omitted, the datastore backend decides how many entities to + send. + + There is an internal buffer for use with the next() method. If this buffer + is not empty, up to 'count' values are removed from this buffer and + returned. It's best not to mix _Next() and next(). + + The results are always returned as a list. If there are no results left, + an empty list is returned. + + Args: + # the number of results to return; must be >= 1 + count: int or long or None + + Returns: + # a list of entities or keys + [Entity or Key, ...] + """ + if count is not None and (not isinstance(count, (int, long)) or count <= 0): raise datastore_errors.BadArgumentError( 'Argument to _Next must be an int greater than 0; received %s (a %s)' % (count, typename(count))) if self.__buffer: - raise datastore_errors.BadRequestError( - 'You can\'t mix next() and _Next()') + if count is None: + entity_list = self.__buffer + self.__buffer = [] + return entity_list + elif count <= len(self.__buffer): + entity_list = self.__buffer[:count] + del self.__buffer[:count] + return entity_list + else: + entity_list = self.__buffer + self.__buffer = [] + count -= len(entity_list) + else: + entity_list = [] + if not self.__more_results: - return [] + return entity_list req = datastore_pb.NextRequest() - req.set_count(count) - req.mutable_cursor().CopyFrom(self._ToPb()) + if count is not None: + req.set_count(count) + req.mutable_cursor().CopyFrom(self.__cursor) result = datastore_pb.QueryResult() try: apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Next', req, result) except apiproxy_errors.ApplicationError, err: raise _ToDatastoreError(err) - self.__more_results = result.more_results() + return entity_list + self._ProcessQueryResult(result) - ret = [Entity._FromPb(r) for r in result.result_list()] - return ret + def _ProcessQueryResult(self, result): + """Returns all results from datastore_pb.QueryResult and updates + self.__more_results - _BUFFER_SIZE = 20 + Not intended to be used by application developers. Use the python + iterator protocol instead. + + The results are always returned as a list. If there are no results left, + an empty list is returned. + + Args: + # the instance of datastore_pb.QueryResult to be stored + result: datastore_pb.QueryResult + + Returns: + # a list of entities or keys + [Entity or Key, ...] + """ + self.__more_results = result.more_results() + + if self.__keys_only: + return [Key._FromPb(e.key()) for e in result.result_list()] + else: + return [Entity._FromPb(e) for e in result.result_list()] def next(self): if not self.__buffer: - self.__buffer = self._Next(self._BUFFER_SIZE) + self.__buffer = self._Next(self.__batch_size) try: return self.__buffer.pop(0) except IndexError: @@ -1253,86 +1837,55 @@ class Iterator(object): def __iter__(self): return self - def _ToPb(self): - """Converts this Iterator to its protocol buffer representation. Not - intended to be used by application developers. Enforced by hiding the - datastore_pb classes. - - Returns: - # the PB representation of this Iterator - datastore_pb.Cursor - """ - pb = datastore_pb.Cursor() - pb.set_cursor(self.__cursor) - return pb - - @staticmethod - def _FromPb(pb): - """Static factory method. Returns the Iterator representation of the given - protocol buffer (datastore_pb.Cursor). Not intended to be used by - application developers. Enforced by not hiding the datastore_pb classes. - - Args: - # a protocol buffer Cursor - pb: datastore_pb.Cursor - - Returns: - # the Iterator representation of the argument - Iterator - """ - return Iterator(pb.cursor()) - - class _Transaction(object): """Encapsulates a transaction currently in progress. - If we've sent a BeginTransaction call, then handle will be a - datastore_pb.Transaction that holds the transaction handle. - If we know the entity group for this transaction, it's stored in the - entity_group attribute, which is set by RecordModifiedKeys(). + entity_group attribute, which is set by RunInTransaction(). modified_keys is a set containing the Keys of all entities modified (ie put or deleted) in this transaction. If an entity is modified more than once, a BadRequestError is raised. """ - def __init__(self): - """Initializes modified_keys to the empty set.""" - self.handle = None + def __init__(self, handle): + """Initializes the transaction. + + Args: + handle: a datastore_pb.Transaction returned by a BeginTransaction call + """ + assert isinstance(handle, datastore_pb.Transaction) + explanation = [] + assert handle.IsInitialized(explanation), explanation + + self.handle = handle self.entity_group = None self.modified_keys = None self.modified_keys = set() - def RecordModifiedKeys(self, keys, error_on_repeat=True): - """Updates the modified keys seen so far. - Also sets entity_group if it hasn't yet been set. +def RunInTransaction(function, *args, **kwargs): + """Runs a function inside a datastore transaction. - If error_on_repeat is True and any of the given keys have already been - modified, raises BadRequestError. + Runs the user-provided function inside transaction, retries default + number of times. Args: - keys: sequence of Keys - """ - keys, _ = NormalizeAndTypeCheckKeys(keys) - - if keys and not self.entity_group: - self.entity_group = keys[0].entity_group() - - keys = set(keys) - - if error_on_repeat: - already_modified = self.modified_keys.intersection(keys) - if already_modified: - raise datastore_errors.BadRequestError( - "Can't update entity more than once in a transaction: %r" % - already_modified.pop()) + # a function to be run inside the transaction + function: callable + # positional arguments to pass to the function + args: variable number of any type - self.modified_keys.update(keys) + Returns: + the function's return value, if any + Raises: + TransactionFailedError, if the transaction could not be committed. + """ + return RunInTransactionCustomRetries( + DEFAULT_TRANSACTION_RETRIES, function, *args, **kwargs) -def RunInTransaction(function, *args, **kwargs): +def RunInTransactionCustomRetries(retries, function, *args, **kwargs): """Runs a function inside a datastore transaction. Runs the user-provided function inside a full-featured, ACID datastore @@ -1387,6 +1940,8 @@ def RunInTransaction(function, *args, **kwargs): Nested transactions are not supported. Args: + # number of retries + retries: integer # a function to be run inside the transaction function: callable # positional arguments to pass to the function @@ -1403,30 +1958,39 @@ def RunInTransaction(function, *args, **kwargs): raise datastore_errors.BadRequestError( 'Nested transactions are not supported.') + if retries < 0: + raise datastore_errors.BadRequestError( + 'Number of retries should be non-negative number.') + tx_key = None try: tx_key = _NewTransactionKey() - tx = _Transaction() - _txes[tx_key] = tx - for i in range(0, TRANSACTION_RETRIES + 1): - tx.modified_keys.clear() + for i in range(0, retries + 1): + handle = datastore_pb.Transaction() + try: + apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction', + api_base_pb.VoidProto(), handle) + except apiproxy_errors.ApplicationError, err: + raise _ToDatastoreError(err) + + tx = _Transaction(handle) + _txes[tx_key] = tx try: result = function(*args, **kwargs) except: original_exception = sys.exc_info() - if tx.handle: - try: - resp = api_base_pb.VoidProto() - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Rollback', - tx.handle, resp) - except: - exc_info = sys.exc_info() - logging.info('Exception sending Rollback:\n' + - ''.join(traceback.format_exception(*exc_info))) + try: + resp = api_base_pb.VoidProto() + apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Rollback', + tx.handle, resp) + except: + exc_info = sys.exc_info() + logging.info('Exception sending Rollback:\n' + + ''.join(traceback.format_exception(*exc_info))) type, value, trace = original_exception if type is datastore_errors.Rollback: @@ -1434,21 +1998,20 @@ def RunInTransaction(function, *args, **kwargs): else: raise type, value, trace - if tx.handle: - try: - resp = api_base_pb.VoidProto() - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit', - tx.handle, resp) - except apiproxy_errors.ApplicationError, err: - if (err.application_error == - datastore_pb.Error.CONCURRENT_TRANSACTION): - logging.warning('Transaction collision for entity group with ' - 'key %r. Retrying...', tx.entity_group) - tx.handle = None - tx.entity_group = None - continue - else: - raise _ToDatastoreError(err) + try: + resp = datastore_pb.CommitResponse() + apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit', + tx.handle, resp) + except apiproxy_errors.ApplicationError, err: + if (err.application_error == + datastore_pb.Error.CONCURRENT_TRANSACTION): + logging.warning('Transaction collision for entity group with ' + 'key %r. Retrying...', tx.entity_group) + tx.handle = None + tx.entity_group = None + continue + else: + raise _ToDatastoreError(err) return result @@ -1462,25 +2025,26 @@ def RunInTransaction(function, *args, **kwargs): def _MaybeSetupTransaction(request, keys): - """Begins a transaction, if necessary, and populates it in the request. + """If we're in a transaction, validates and populates it in the request. If we're currently inside a transaction, this records the entity group, - checks that the keys are all in that entity group, creates the transaction - PB, and sends the BeginTransaction. It then populates the transaction handle - in the request. + checks that the keys are all in that entity group, and populates the + transaction handle in the request. Raises BadRequestError if the entity has a different entity group than the current transaction. Args: - request: GetRequest, PutRequest, or DeleteRequest + request: GetRequest, PutRequest, DeleteRequest, or Query keys: sequence of Keys Returns: _Transaction if we're inside a transaction, otherwise None """ assert isinstance(request, (datastore_pb.GetRequest, datastore_pb.PutRequest, - datastore_pb.DeleteRequest)) + datastore_pb.DeleteRequest, datastore_pb.Query, + taskqueue_service_pb.TaskQueueAddRequest, + )), request.__class__ tx_key = None try: @@ -1491,8 +2055,11 @@ def _MaybeSetupTransaction(request, keys): groups = [k.entity_group() for k in keys] if tx.entity_group: expected_group = tx.entity_group - else: + elif groups: expected_group = groups[0] + else: + expected_group = None + for group in groups: if (group != expected_group or @@ -1505,12 +2072,10 @@ def _MaybeSetupTransaction(request, keys): (not group.has_id_or_name() and group is not expected_group)): raise _DifferentEntityGroupError(expected_group, group) - if not tx.handle: - tx.handle = datastore_pb.Transaction() - req = api_base_pb.VoidProto() - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction', req, - tx.handle) + if not tx.entity_group and group.has_id_or_name(): + tx.entity_group = group + assert tx.handle.IsInitialized() request.mutable_transaction().CopyFrom(tx.handle) return tx @@ -1544,7 +2109,7 @@ def _FindTransactionFrameInStack(): """Walks the stack to find a RunInTransaction() call. Returns: - # this is the RunInTransaction() frame record, if found + # this is the RunInTransactionCustomRetries() frame record, if found frame record or None """ frame = sys._getframe() @@ -1553,7 +2118,7 @@ def _FindTransactionFrameInStack(): frame = frame.f_back.f_back while frame: if (frame.f_code.co_filename == filename and - frame.f_code.co_name == 'RunInTransaction'): + frame.f_code.co_name == 'RunInTransactionCustomRetries'): return frame frame = frame.f_back @@ -1592,6 +2157,28 @@ def _GetCompleteKeyOrError(arg): return key +def _GetPropertyValue(entity, property): + """Returns an entity's value for a given property name. + + Handles special properties like __key__ as well as normal properties. + + Args: + entity: datastore.Entity + property: str; the property name + + Returns: + property value. For __key__, a datastore_types.Key. + + Raises: + KeyError, if the entity does not have the given property. + """ + if property in datastore_types._SPECIAL_PROPERTIES: + assert property == datastore_types._KEY_SPECIAL_PROPERTY + return entity.key() + else: + return entity[property] + + def _AddOrAppend(dictionary, key, value): """Adds the value to the existing values in the dictionary, if any. @@ -1634,6 +2221,6 @@ def _ToDatastoreError(err): } if err.application_error in errors: - raise errors[err.application_error](err.error_detail) + return errors[err.application_error](err.error_detail) else: - raise datastore_errors.Error(err.error_detail) + return datastore_errors.Error(err.error_detail) google_appengine/google/appengine/api/datastore.py @@ -39,7 +39,7 @@ _DIRECTION_MAP = { } -def GetSchema(_app=None): +def GetSchema(_app=None, properties=True, start_kind=None, end_kind=None): """Infers an app's schema from the entities in the datastore. Note that the PropertyValue PBs in the returned EntityProtos are empty @@ -48,11 +48,21 @@ def GetSchema(_app=None): throw UserNotFoundError because their email and auth domain fields will be empty. + Args: + properties: boolean, whether to include property names and types + start_kind, end_kind: optional range endpoints for the kinds to return, + compared lexicographically + Returns: list of entity_pb.EntityProto, with kind and property names and types """ - req = api_base_pb.StringProto() - req.set_value(datastore_types.ResolveAppId(_app)) + req = datastore_pb.GetSchemaRequest() + req.set_app(datastore_types.ResolveAppId(_app)) + req.set_properties(properties) + if start_kind is not None: + req.set_start_kind(start_kind) + if end_kind is not None: + req.set_end_kind(end_kind) resp = datastore_pb.Schema() _Call('GetSchema', req, resp) google_appengine/google/appengine/api/datastore_admin.py @@ -37,6 +37,7 @@ per-transaction, so they should only be used by one tx at a time. import datetime import logging +import md5 import os import struct import sys @@ -74,6 +75,7 @@ _MAX_QUERY_OFFSET = 1000 _MAX_QUERY_COMPONENTS = 100 +_BATCH_SIZE = 20 class _StoredEntity(object): """Simple wrapper around an entity stored by the stub. @@ -96,6 +98,91 @@ class _StoredEntity(object): self.native = datastore.Entity._FromPb(entity) +class _Cursor(object): + """A query cursor. + + Public properties: + cursor: the integer cursor + count: the original total number of results + keys_only: whether the query is keys_only + + Class attributes: + _next_cursor: the next cursor to allocate + _next_cursor_lock: protects _next_cursor + _offset: the internal index for where we are in the results + """ + _next_cursor = 1 + _next_cursor_lock = threading.Lock() + + def __init__(self, results, keys_only): + """Constructor. + + Args: + # the query results, in order, such that results[self.offset:] is + # the next result + results: list of entity_pb.EntityProto + keys_only: integer + """ + self.__results = results + self.count = len(results) + self.keys_only = keys_only + self._offset = 0 + + self._next_cursor_lock.acquire() + try: + self.cursor = _Cursor._next_cursor + _Cursor._next_cursor += 1 + finally: + self._next_cursor_lock.release() + + def PopulateQueryResult(self, result, count, offset=None, compiled=False): + """Populates a QueryResult with this cursor and the given number of results. + + Args: + result: datastore_pb.QueryResult + count: integer + offset: integer, overrides the internal offset + compiled: boolean, whether we are compiling this query + """ + _offset = offset + if offset is None: + _offset = self._offset + if count > _MAXIMUM_RESULTS: + count = _MAXIMUM_RESULTS + + result.mutable_cursor().set_cursor(self.cursor) + result.set_keys_only(self.keys_only) + + self.count = len(self.__results[_offset:_offset + count]) + + results_pbs = [r._ToPb() for r in self.__results[_offset:_offset + count]] + result.result_list().extend(results_pbs) + + if offset is None: + self._offset += self.count + + result.set_more_results(len(self.__results[_offset + self.count:]) > 0) + if compiled and result.more_results(): + compiled_query = _FakeCompiledQuery(cursor=self.cursor, + offset=_offset + self.count, + keys_only=self.keys_only) + result.mutable_compiled_query().CopyFrom(compiled_query._ToPb()) + + +class _FakeCompiledQuery(object): + def __init__(self, cursor, offset, keys_only=False): + self.cursor = cursor + self.offset = offset + self.keys_only = keys_only + + def _ToPb(self): + compiled_pb = datastore_pb.CompiledQuery() + compiled_pb.mutable_primaryscan() + compiled_pb.set_keys_only(self.keys_only) + + compiled_pb.set_limit(self.cursor) + compiled_pb.set_offset(self.offset) + return compiled_pb class DatastoreFileStub(apiproxy_stub.APIProxyStub): """ Persistent stub for the Python datastore API. @@ -128,12 +215,25 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): users.User: entity_pb.PropertyValue.kUserValueGroup, } + WRITE_ONLY = entity_pb.CompositeIndex.WRITE_ONLY + READ_WRITE = entity_pb.CompositeIndex.READ_WRITE + DELETED = entity_pb.CompositeIndex.DELETED + ERROR = entity_pb.CompositeIndex.ERROR + + _INDEX_STATE_TRANSITIONS = { + WRITE_ONLY: frozenset((READ_WRITE, DELETED, ERROR)), + READ_WRITE: frozenset((DELETED,)), + ERROR: frozenset((DELETED,)), + DELETED: frozenset((ERROR,)), + } + def __init__(self, app_id, datastore_file, - history_file, + history_file=None, require_indexes=False, - service_name='datastore_v3'): + service_name='datastore_v3', + trusted=False): """Constructor. Initializes and loads the datastore from the backing files, if they exist. @@ -142,11 +242,12 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): app_id: string datastore_file: string, stores all entities across sessions. Use None not to use a file. - history_file: string, stores query history. Use None as with - datastore_file. + history_file: DEPRECATED. No-op. require_indexes: bool, default False. If True, composite indexes must exist in index.yaml for queries that need them. service_name: Service name expected for all calls. + trusted: bool, default False. If True, this stub allows an app to + access the data of another app. """ super(DatastoreFileStub, self).__init__(service_name) @@ -154,7 +255,7 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): assert isinstance(app_id, basestring) and app_id != '' self.__app_id = app_id self.__datastore_file = datastore_file - self.__history_file = history_file + self.SetTrusted(trusted) self.__entities = {} @@ -172,11 +273,9 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__query_history = {} self.__next_id = 1 - self.__next_cursor = 1 self.__next_tx_handle = 1 self.__next_index_id = 1 self.__id_lock = threading.Lock() - self.__cursor_lock = threading.Lock() self.__tx_handle_lock = threading.Lock() self.__index_id_lock = threading.Lock() self.__tx_lock = threading.Lock() @@ -195,7 +294,49 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__query_history = {} self.__schema_cache = {} - def _AppKindForKey(self, key): + def SetTrusted(self, trusted): + """Set/clear the trusted bit in the stub. + + This bit indicates that the app calling the stub is trusted. A + trusted app can write to datastores of other apps. + + Args: + trusted: boolean. + """ + self.__trusted = trusted + + def __ValidateAppId(self, app_id): + """Verify that this is the stub for app_id. + + Args: + app_id: An application ID. + + Raises: + datastore_errors.BadRequestError: if this is not the stub for app_id. + """ + if not self.__trusted and app_id != self.__app_id: + raise datastore_errors.BadRequestError( + 'app %s cannot access app %s\'s data' % (self.__app_id, app_id)) + + def __ValidateKey(self, key): + """Validate this key. + + Args: + key: entity_pb.Reference + + Raises: + datastore_errors.BadRequestError: if the key is invalid + """ + assert isinstance(key, entity_pb.Reference) + + self.__ValidateAppId(key.app()) + + for elem in key.path().element_list(): + if elem.has_id() == elem.has_name(): + raise datastore_errors.BadRequestError( + 'each key path element should have id or name but not both: %r' % key) + + def _AppIdNamespaceKindForKey(self, key): """ Get (app, kind) tuple from given key. The (app, kind) tuple is used as an index into several internal @@ -217,7 +358,7 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): entity: entity_pb.EntityProto """ key = entity.key() - app_kind = self._AppKindForKey(key) + app_kind = self._AppIdNamespaceKindForKey(key) if app_kind not in self.__entities: self.__entities[app_kind] = {} self.__entities[app_kind][key] = _StoredEntity(entity) @@ -271,25 +412,11 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): if last_path.has_id() and last_path.id() >= self.__next_id: self.__next_id = last_path.id() + 1 - self.__query_history = {} - for encoded_query, count in self.__ReadPickled(self.__history_file): - try: - query_pb = datastore_pb.Query(encoded_query) - except self.READ_PB_EXCEPTIONS, e: - raise datastore_errors.InternalError(self.READ_ERROR_MSG % - (self.__history_file, e)) - - if query_pb in self.__query_history: - self.__query_history[query_pb] += count - else: - self.__query_history[query_pb] = count - def Write(self): """ Writes out the datastore and history files. Be careful! If the files already exist, this method overwrites them! """ self.__WriteDatastore() - self.__WriteHistory() def __WriteDatastore(self): """ Writes out the datastore file. Be careful! If the file already exist, @@ -303,16 +430,6 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__WritePickled(encoded, self.__datastore_file) - def __WriteHistory(self): - """ Writes out the history file. Be careful! If the file already exist, - this method overwrites it! - """ - if self.__history_file and self.__history_file != '/dev/null': - encoded = [(query.Encode(), count) - for query, count in self.__query_history.items()] - - self.__WritePickled(encoded, self.__history_file) - def __ReadPickled(self, filename): """Reads a pickled object from the given file and returns it. """ @@ -324,7 +441,7 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): return pickle.load(open(filename, 'rb')) else: logging.warning('Could not read datastore data from %s', filename) - except (AttributeError, LookupError, NameError, TypeError, + except (AttributeError, LookupError, ImportError, NameError, TypeError, ValueError, struct.error, pickle.PickleError), e: raise datastore_errors.InternalError( 'Could not read data from %s. Try running with the ' @@ -362,16 +479,20 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__file_lock.release() def MakeSyncCall(self, service, call, request, response): - """ The main RPC entry point. service must be 'datastore_v3'. So far, the - supported calls are 'Get', 'Put', 'RunQuery', 'Next', and 'Count'. + """ The main RPC entry point. service must be 'datastore_v3'. """ + self.assertPbIsInitialized(request) super(DatastoreFileStub, self).MakeSyncCall(service, call, request, response) + self.assertPbIsInitialized(response) + def assertPbIsInitialized(self, pb): + """Raises an exception if the given PB is not initialized and valid.""" explanation = [] - assert response.IsInitialized(explanation), explanation + assert pb.IsInitialized(explanation), explanation + pb.Encode() def QueryHistory(self): """Returns a dict that maps Query PBs to times they've been run. @@ -382,8 +503,18 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): def _Dynamic_Put(self, put_request, put_response): clones = [] for entity in put_request.entity_list(): + self.__ValidateKey(entity.key()) + clone = entity_pb.EntityProto() clone.CopyFrom(entity) + + for property in clone.property_list(): + if property.value().has_uservalue(): + uid = md5.new(property.value().uservalue().email().lower()).digest() + uid = '1' + ''.join(['%02d' % ord(x) for x in uid])[:20] + property.mutable_value().mutable_uservalue().set_obfuscated_gaiaid( + uid) + clones.append(clone) assert clone.has_key() @@ -420,24 +551,31 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): def _Dynamic_Get(self, get_request, get_response): + if get_request.has_transaction(): + entities = self.__tx_snapshot + else: + entities = self.__entities + for key in get_request.key_list(): - app_kind = self._AppKindForKey(key) + self.__ValidateAppId(key.app()) + app_kind = self._AppIdNamespaceKindForKey(key) - group = get_response.add_entity() - try: - entity = self.__entities[app_kind][key].protobuf - except KeyError: - entity = None + group = get_response.add_entity() + try: + entity = entities[app_kind][key].protobuf + except KeyError: + entity = None - if entity: - group.mutable_entity().CopyFrom(entity) + if entity: + group.mutable_entity().CopyFrom(entity) def _Dynamic_Delete(self, delete_request, delete_response): self.__entities_lock.acquire() try: for key in delete_request.key_list(): - app_kind = self._AppKindForKey(key) + self.__ValidateAppId(key.app()) + app_kind = self._AppIdNamespaceKindForKey(key) try: del self.__entities[app_kind][key] if not self.__entities[app_kind]: @@ -453,13 +591,21 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__entities_lock.release() - def _Dynamic_RunQuery(self, query, query_result): + def _Dynamic_RunQuery(self, query, query_result, count=None): if not self.__tx_lock.acquire(False): - raise apiproxy_errors.ApplicationError( - datastore_pb.Error.BAD_REQUEST, 'Can\'t query inside a transaction.') + if not query.has_ancestor(): + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, + 'Only ancestor queries are allowed inside transactions.') + entities = self.__tx_snapshot else: + entities = self.__entities self.__tx_lock.release() + app_id_namespace = datastore_types.parse_app_id_namespace(query.app()) + app_id = app_id_namespace.app_id() + self.__ValidateAppId(app_id) + if query.has_offset() and query.offset() > _MAX_QUERY_OFFSET: raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'Too big query offset.') @@ -473,13 +619,14 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): ('query is too large. may not have more than %s filters' ' + sort orders ancestor total' % _MAX_QUERY_COMPONENTS)) - app = query.app() + (filters, orders) = datastore_index.Normalize(query.filter_list(), + query.order_list()) if self.__require_indexes: required, kind, ancestor, props, num_eq_filters = datastore_index.CompositeIndexForQuery(query) if required: required_key = kind, ancestor, props - indexes = self.__indexes.get(app) + indexes = self.__indexes.get(app_id) if not indexes: raise apiproxy_errors.ApplicationError( datastore_pb.Error.NEED_INDEX, @@ -506,9 +653,15 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): "You must update the index.yaml file in your application root.") try: - query.set_app(app) - results = self.__entities[app, query.kind()].values() - results = [entity.native for entity in results] + query.set_app(app_id_namespace.to_encoded()) + if query.has_kind(): + results = entities[app_id_namespace.to_encoded(), query.kind()].values() + results = [entity.native for entity in results] + else: + results = [] + for key in entities: + if key[0] == app_id_namespace.to_encoded(): + results += [entity.native for entity in entities[key].values()] except KeyError: results = [] @@ -526,7 +679,23 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): datastore_pb.Query_Filter.EQUAL: '==', } - for filt in query.filter_list(): + def has_prop_indexed(entity, prop): + """Returns True if prop is in the entity and is indexed.""" + if prop in datastore_types._SPECIAL_PROPERTIES: + return True + elif prop in entity.unindexed_properties(): + return False + + values = entity.get(prop, []) + if not isinstance(values, (tuple, list)): + values = [values] + + for value in values: + if type(value) not in datastore_types._RAW_PROPERTY_TYPES: + return True + return False + + for filt in filters: assert filt.op() != datastore_pb.Query_Filter.IN prop = filt.property(0).name().decode('utf-8') @@ -535,20 +704,24 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): filter_val_list = [datastore_types.FromPropertyPb(filter_prop) for filter_prop in filt.property_list()] - def passes(entity): - """ Returns True if the entity passes the filter, False otherwise. """ - if prop in datastore_types._SPECIAL_PROPERTIES: - entity_vals = self.__GetSpecialPropertyValue(entity, prop) - else: - entity_vals = entity.get(prop, []) + def passes_filter(entity): + """Returns True if the entity passes the filter, False otherwise. + + The filter being evaluated is filt, the current filter that we're on + in the list of filters in the query. + """ + if not has_prop_indexed(entity, prop): + return False + + try: + entity_vals = datastore._GetPropertyValue(entity, prop) + except KeyError: + entity_vals = [] if not isinstance(entity_vals, list): entity_vals = [entity_vals] for fixed_entity_val in entity_vals: - if type(fixed_entity_val) in datastore_types._RAW_PROPERTY_TYPES: - continue - for filter_val in filter_val_list: fixed_entity_type = self._PROPERTY_TYPE_TAGS.get( fixed_entity_val.__class__) @@ -572,24 +745,9 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): return False - results = filter(passes, results) - - def has_prop_indexed(entity, prop): - """Returns True if prop is in the entity and is not a raw property, or - is a special property.""" - if prop in datastore_types._SPECIAL_PROPERTIES: - return True - - values = entity.get(prop, []) - if not isinstance(values, (tuple, list)): - values = [values] - - for value in values: - if type(value) not in datastore_types._RAW_PROPERTY_TYPES: - return True - return False + results = filter(passes_filter, results) - for order in query.order_list(): + for order in orders: prop = order.property().decode('utf-8') results = [entity for entity in results if has_prop_indexed(entity, prop)] @@ -598,22 +756,18 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): entity a is considered smaller than, equal to, or larger than b, according to the query's orderings. """ cmped = 0 - for o in query.order_list(): + for o in orders: prop = o.property().decode('utf-8') reverse = (o.direction() is datastore_pb.Query_Order.DESCENDING) - if prop in datastore_types._SPECIAL_PROPERTIES: - a_val = self.__GetSpecialPropertyValue(a, prop) - b_val = self.__GetSpecialPropertyValue(b, prop) - else: - a_val = a[prop] - if isinstance(a_val, list): - a_val = sorted(a_val, order_compare_properties, reverse=reverse)[0] + a_val = datastore._GetPropertyValue(a, prop) + if isinstance(a_val, list): + a_val = sorted(a_val, order_compare_properties, reverse=reverse)[0] - b_val = b[prop] - if isinstance(b_val, list): - b_val = sorted(b_val, order_compare_properties, reverse=reverse)[0] + b_val = datastore._GetPropertyValue(b, prop) + if isinstance(b_val, list): + b_val = sorted(b_val, order_compare_properties, reverse=reverse)[0] cmped = order_compare_properties(a_val, b_val) @@ -659,7 +813,7 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): limit = query.limit() if limit > _MAXIMUM_RESULTS: limit = _MAXIMUM_RESULTS - results = results[offset:limit + offset] + results = results[offset:] clone = datastore_pb.Query() clone.CopyFrom(query) @@ -668,39 +822,59 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__query_history[clone] += 1 else: self.__query_history[clone] = 1 - self.__WriteHistory() - self.__cursor_lock.acquire() - cursor = self.__next_cursor - self.__next_cursor += 1 - self.__cursor_lock.release() - self.__queries[cursor] = (results, len(results)) + cursor = _Cursor(results, query.keys_only()) + self.__queries[cursor.cursor] = cursor - query_result.mutable_cursor().set_cursor(cursor) - query_result.set_more_results(len(results) > 0) + if count is None: + if query.has_count(): + count = query.count() + elif query.has_limit(): + count = query.limit() + else: + count = _BATCH_SIZE - def _Dynamic_Next(self, next_request, query_result): - cursor = next_request.cursor().cursor() + cursor.PopulateQueryResult(query_result, count, compiled=query.compile()) + + def _Dynamic_RunCompiledQuery(self, compiled_request, query_result): + cursor_handle = compiled_request.compiled_query().limit() + cursor_offset = compiled_request.compiled_query().offset() try: - results, orig_count = self.__queries[cursor] + cursor = self.__queries[cursor_handle] except KeyError: - raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, - 'Cursor %d not found' % cursor) + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, 'Cursor %d not found' % cursor_handle) - count = next_request.count() - results_pb = [r._ToPb() for r in results[:count]] - query_result.result_list().extend(results_pb) - del results[:count] + count = _BATCH_SIZE + if compiled_request.has_count(): + count = compiled_request.count() + cursor.PopulateQueryResult( + query_result, count, cursor_offset, compiled=True) - query_result.set_more_results(len(results) > 0) + def _Dynamic_Next(self, next_request, query_result): + cursor_handle = next_request.cursor().cursor() + + try: + cursor = self.__queries[cursor_handle] + except KeyError: + raise apiproxy_errors.ApplicationError( + datastore_pb.Error.BAD_REQUEST, 'Cursor %d not found' % cursor_handle) + + count = _BATCH_SIZE + if next_request.has_count(): + count = next_request.count() + cursor.PopulateQueryResult(query_result, count) def _Dynamic_Count(self, query, integer64proto): + self.__ValidateAppId(query.app()) query_result = datastore_pb.QueryResult() - self._Dynamic_RunQuery(query, query_result) + count = _MAXIMUM_RESULTS + if query.has_limit(): + count = query.limit() + self._Dynamic_RunQuery(query, query_result, count=count) cursor = query_result.cursor().cursor() - results, count = self.__queries[cursor] - integer64proto.set_value(count) + integer64proto.set_value(self.__queries[cursor].count) del self.__queries[cursor] def _Dynamic_BeginTransaction(self, request, transaction): @@ -739,70 +913,97 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__tx_snapshot = {} self.__tx_lock.release() - def _Dynamic_GetSchema(self, app_str, schema): - minint = -sys.maxint - 1 - try: - minfloat = float('-inf') - except ValueError: - minfloat = -1e300000 - - app_str = app_str.value() + def _Dynamic_GetSchema(self, req, schema): + app_str = req.app() + self.__ValidateAppId(app_str) kinds = [] for app, kind in self.__entities: - if app == app_str: - app_kind = (app, kind) - if app_kind in self.__schema_cache: - kinds.append(self.__schema_cache[app_kind]) - continue - - kind_pb = entity_pb.EntityProto() - kind_pb.mutable_key().set_app('') - kind_pb.mutable_key().mutable_path().add_element().set_type(kind) - kind_pb.mutable_entity_group() - - props = {} - - for entity in self.__entities[app_kind].values(): - for prop in entity.protobuf.property_list(): - if prop.name() not in props: - props[prop.name()] = entity_pb.PropertyValue() - props[prop.name()].MergeFrom(prop.value()) - - for value_pb in props.values(): - if value_pb.has_int64value(): - value_pb.set_int64value(minint) - if value_pb.has_booleanvalue(): - value_pb.set_booleanvalue(False) - if value_pb.has_stringvalue(): - value_pb.set_stringvalue('') - if value_pb.has_doublevalue(): - value_pb.set_doublevalue(minfloat) - if value_pb.has_pointvalue(): - value_pb.mutable_pointvalue().set_x(minfloat) - value_pb.mutable_pointvalue().set_y(minfloat) - if value_pb.has_uservalue(): - value_pb.mutable_uservalue().set_gaiaid(minint) - value_pb.mutable_uservalue().set_email('') - value_pb.mutable_uservalue().set_auth_domain('') - value_pb.mutable_uservalue().clear_nickname() - elif value_pb.has_referencevalue(): - value_pb.clear_referencevalue() - value_pb.mutable_referencevalue().set_app('') - - for name, value_pb in props.items(): - prop_pb = kind_pb.add_property() - prop_pb.set_name(name) - prop_pb.mutable_value().CopyFrom(value_pb) - - kinds.append(kind_pb) - self.__schema_cache[app_kind] = kind_pb + if (app != app_str or + (req.has_start_kind() and kind < req.start_kind()) or + (req.has_end_kind() and kind > req.end_kind())): + continue + + app_kind = (app, kind) + if app_kind in self.__schema_cache: + kinds.append(self.__schema_cache[app_kind]) + continue + + kind_pb = entity_pb.EntityProto() + kind_pb.mutable_key().set_app('') + kind_pb.mutable_key().mutable_path().add_element().set_type(kind) + kind_pb.mutable_entity_group() + + props = {} + + for entity in self.__entities[app_kind].values(): + for prop in entity.protobuf.property_list(): + if prop.name() not in props: + props[prop.name()] = entity_pb.PropertyValue() + props[prop.name()].MergeFrom(prop.value()) + + for value_pb in props.values(): + if value_pb.has_int64value(): + value_pb.set_int64value(0) + if value_pb.has_booleanvalue(): + value_pb.set_booleanvalue(False) + if value_pb.has_stringvalue(): + value_pb.set_stringvalue('none') + if value_pb.has_doublevalue(): + value_pb.set_doublevalue(0.0) + if value_pb.has_pointvalue(): + value_pb.mutable_pointvalue().set_x(0.0) + value_pb.mutable_pointvalue().set_y(0.0) + if value_pb.has_uservalue(): + value_pb.mutable_uservalue().set_gaiaid(0) + value_pb.mutable_uservalue().set_email('none') + value_pb.mutable_uservalue().set_auth_domain('none') + value_pb.mutable_uservalue().clear_nickname() + value_pb.mutable_uservalue().clear_obfuscated_gaiaid() + if value_pb.has_referencevalue(): + value_pb.clear_referencevalue() + value_pb.mutable_referencevalue().set_app('none') + pathelem = value_pb.mutable_referencevalue().add_pathelement() + pathelem.set_type('none') + pathelem.set_name('none') + + for name, value_pb in props.items(): + prop_pb = kind_pb.add_property() + prop_pb.set_name(name) + prop_pb.set_multiple(False) + prop_pb.mutable_value().CopyFrom(value_pb) + + kinds.append(kind_pb) + self.__schema_cache[app_kind] = kind_pb for kind_pb in kinds: - schema.add_kind().CopyFrom(kind_pb) + kind = schema.add_kind() + kind.CopyFrom(kind_pb) + if not req.properties(): + kind.clear_property() + + schema.set_more_results(False) + + def _Dynamic_AllocateIds(self, allocate_ids_request, allocate_ids_response): + model_key = allocate_ids_request.model_key() + size = allocate_ids_request.size() + + self.__ValidateAppId(model_key.app()) + + try: + self.__id_lock.acquire() + start = self.__next_id + self.__next_id += size + end = self.__next_id - 1 + finally: + self.__id_lock.release() + + allocate_ids_response.set_start(start) + allocate_ids_response.set_end(end) def _Dynamic_CreateIndex(self, index, id_response): + self.__ValidateAppId(index.app_id()) if index.id() != 0: raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, 'New index id must be 0.') @@ -830,15 +1031,18 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__indexes_lock.release() def _Dynamic_GetIndices(self, app_str, composite_indices): + self.__ValidateAppId(app_str.value()) composite_indices.index_list().extend( self.__indexes.get(app_str.value(), [])) def _Dynamic_UpdateIndex(self, index, void): + self.__ValidateAppId(index.app_id()) stored_index = self.__FindIndex(index) if not stored_index: raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, "Index doesn't exist.") - elif index.state() != stored_index.state() + 1: + elif (index.state() != stored_index.state() and + index.state() not in self._INDEX_STATE_TRANSITIONS[stored_index.state()]): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, "cannot move index state from %s to %s" % @@ -852,6 +1056,7 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): self.__indexes_lock.release() def _Dynamic_DeleteIndex(self, index, void): + self.__ValidateAppId(index.app_id()) stored_index = self.__FindIndex(index) if not stored_index: raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, @@ -874,29 +1079,10 @@ class DatastoreFileStub(apiproxy_stub.APIProxyStub): entity_pb.CompositeIndex, if it exists; otherwise None """ app = index.app_id() + self.__ValidateAppId(app) if app in self.__indexes: for stored_index in self.__indexes[app]: if index.definition() == stored_index.definition(): return stored_index return None - - @classmethod - def __GetSpecialPropertyValue(cls, entity, property): - """Returns an entity's value for a special property. - - Right now, the only special property is __key__, whose value is the - entity's key. - - Args: - entity: datastore.Entity - - Returns: - property value. For __key__, a datastore_types.Key. - - Raises: - AssertionError, if the given property is not special. - """ - assert property in datastore_types._SPECIAL_PROPERTIES - if property == datastore_types._KEY_SPECIAL_PROPERTY: - return entity.key() google_appengine/google/appengine/api/datastore_file_stub.py @@ -47,6 +47,7 @@ from xml.sax import saxutils from google.appengine.datastore import datastore_pb from google.appengine.api import datastore_errors from google.appengine.api import users +from google.appengine.api import namespace_manager from google.net.proto import ProtocolBuffer from google.appengine.datastore import entity_pb @@ -59,6 +60,8 @@ RESERVED_PROPERTY_NAME = re.compile('^__.*__$') _KEY_SPECIAL_PROPERTY = '__key__' _SPECIAL_PROPERTIES = frozenset([_KEY_SPECIAL_PROPERTY]) +_NAMESPACE_SEPARATOR='!' + class UtcTzinfo(datetime.tzinfo): def utcoffset(self, dt): return datetime.timedelta(0) def dst(self, dt): return datetime.timedelta(0) @@ -80,7 +83,8 @@ def typename(obj): def ValidateString(value, name='unused', exception=datastore_errors.BadValueError, - max_len=_MAX_STRING_LENGTH): + max_len=_MAX_STRING_LENGTH, + empty_ok=False): """Raises an exception if value is not a valid string or a subclass thereof. A string is valid if it's not empty, no more than _MAX_STRING_LENGTH bytes, @@ -91,17 +95,49 @@ def ValidateString(value, value: the value to validate. name: the name of this value; used in the exception message. exception: the type of exception to raise. - max_len: the maximum allowed length, in bytes + max_len: the maximum allowed length, in bytes. + empty_ok: allow empty value. """ + if value is None and empty_ok: + return if not isinstance(value, basestring) or isinstance(value, Blob): raise exception('%s should be a string; received %s (a %s):' % (name, value, typename(value))) - if not value: + if not value and not empty_ok: raise exception('%s must not be empty.' % name) if len(value.encode('utf-8')) > max_len: raise exception('%s must be under %d bytes.' % (name, max_len)) +def ValidateInteger(value, + name='unused', + exception=datastore_errors.BadValueError, + empty_ok=False, + zero_ok=False, + negative_ok=False): + """Raises an exception if value is not a valid integer. + + An integer is valid if it's not negative or empty and is an integer. + The exception type can be specified with the exception argument; + it defaults to BadValueError. + + Args: + value: the value to validate. + name: the name of this value; used in the exception message. + exception: the type of exception to raise. + empty_ok: allow None value. + zero_ok: allow zero value. + negative_ok: allow negative value. + """ + if value is None and empty_ok: + return + if not isinstance(value, int): + raise exception('%s should be an integer; received %s (a %s).' % + (name, value, typename(value))) + if not value and not zero_ok: + raise exception('%s must not be 0 (zero)' % name) + if value < 0 and not negative_ok: + raise exception('%s must not be negative.' % name) def ResolveAppId(app, name='_app'): """Validate app id, providing a default. @@ -124,6 +160,152 @@ def ResolveAppId(app, name='_app'): return app +class AppIdNamespace(object): + """Combined AppId and Namespace + + An identifier that combines the application identifier and the + namespace. + """ + __app_id = None + __namespace = None + + def __init__(self, app_id, namespace): + """Constructor. Creates a AppIdNamespace from two strings. + + Args: + app_id: application identifier string + namespace: namespace identifier string + Raises: + BadArgumentError if the values contain + the _NAMESPACE_SEPARATOR character (!) or + the app_id is empty. + """ + self.__app_id = app_id + if namespace: + self.__namespace = namespace + else: + self.__namespace = None + ValidateString(self.__app_id, 'app_id', datastore_errors.BadArgumentError) + ValidateString(self.__namespace, + 'namespace', datastore_errors.BadArgumentError, + empty_ok=True) + if _NAMESPACE_SEPARATOR in self.__app_id: + raise datastore_errors.BadArgumentError( + 'app_id must not contain a "%s"' % _NAMESPACE_SEPARATOR) + if self.__namespace and _NAMESPACE_SEPARATOR in self.__namespace: + raise datastore_errors.BadArgumentError( + 'namespace must not contain a "%s"' % _NAMESPACE_SEPARATOR) + + def __cmp__(self, other): + """Returns negative, zero, or positive when comparing two AppIdNamespace. + + Args: + other: AppIdNamespace to compare to. + + Returns: + Negative if self is less than "other" + Zero if "other" is equal to self + Positive if self is greater than "other" + """ + if not isinstance(other, AppIdNamespace): + return cmp(id(self), id(other)) + return cmp((self.__app_id, self.__namespace), + (other.__app_id, other.__namespace)) + + def to_encoded(self): + """Returns this AppIdNamespace's string equivalent + + i.e. "app!namespace" + """ + if not self.__namespace: + return self.__app_id + else: + return self.__app_id + _NAMESPACE_SEPARATOR + self.__namespace + + def app_id(self): + """Returns this AppId portion of this AppIdNamespace. + """ + return self.__app_id; + + def namespace(self): + """Returns this namespace portion of this AppIdNamespace. + """ + return self.__namespace; + + +def PartitionString(value, separator): + """Equivalent to python2.5 str.partition() + TODO(gmariani) use str.partition() when python 2.5 is adopted. + + Args: + value: String to be partitioned + separator: Separator string + """ + index = value.find(separator); + if index == -1: + return (value, '', value[0:0]); + else: + return (value[0:index], separator, value[index+len(separator):len(value)]) + + +def parse_app_id_namespace(app_id_namespace): + """ + An app_id_namespace string is valid if it's not empty, and contains + at most one namespace separator ('!'). Also, an app_id_namespace + with an empty namespace must not contain a namespace separator. + + Args: + app_id_namespace: an encoded app_id_namespace. + Raises exception if format of app_id_namespace is invalid. + """ + if not app_id_namespace: + raise datastore_errors.BadArgumentError( + 'app_id_namespace must be non empty') + parts = PartitionString(app_id_namespace, _NAMESPACE_SEPARATOR) + if parts[1] == _NAMESPACE_SEPARATOR: + if not parts[2]: + raise datastore_errors.BadArgumentError( + 'app_id_namespace must not contain a "%s" if the namespace is empty' % + _NAMESPACE_SEPARATOR) + if parts[2]: + return AppIdNamespace(parts[0], parts[2]) + return AppIdNamespace(parts[0], None) + +def ResolveAppIdNamespace( + app_id=None, namespace=None, app_id_namespace=None): + """Validate an app id/namespace and substitute default values. + + If the argument is None, $APPLICATION_ID!$NAMESPACE is substituted. + + Args: + app_id: The app id argument value to be validated. + namespace: The namespace argument value to be validated. + app_id_namespace: An AppId/Namespace pair + + Returns: + An AppIdNamespace object initialized with AppId and Namespace. + + Raises: + BadArgumentError if the value is empty or not a string. + """ + if app_id_namespace is None: + if app_id is None: + app_id = os.environ.get('APPLICATION_ID', '') + if namespace is None: + namespace = namespace_manager.get_request_namespace(); + else: + if not app_id is None: + raise datastore_errors.BadArgumentError( + 'app_id is overspecified. Cannot define app_id_namespace and app_id') + if not namespace is None: + raise datastore_errors.BadArgumentError( + 'namespace is overspecified. ' + + 'Cannot define app_id_namespace and namespace') + return parse_app_id_namespace(app_id_namespace) + + return AppIdNamespace(app_id, namespace) + + class Key(object): """The primary key for a datastore entity. @@ -143,6 +325,7 @@ class Key(object): # a base64-encoded primary key, generated by Key.__str__ encoded: str """ + self._str = None if encoded is not None: if not isinstance(encoded, basestring): try: @@ -157,10 +340,13 @@ class Key(object): if modulo != 0: encoded += ('=' * (4 - modulo)) - encoded_pb = base64.urlsafe_b64decode(str(encoded)) + self._str = str(encoded) + encoded_pb = base64.urlsafe_b64decode(self._str) self.__reference = entity_pb.Reference(encoded_pb) assert self.__reference.IsInitialized() + self._str = self._str.rstrip('=') + except (AssertionError, TypeError), e: raise datastore_errors.BadKeyError( 'Invalid string key %s. Details: %s' % (encoded, e)) @@ -172,6 +358,29 @@ class Key(object): else: self.__reference = entity_pb.Reference() + def to_path(self, _default_id=None): + """Construct the "path" of this key as a list. + + Returns: + A list [kind_1, id_or_name_1, ..., kind_n, id_or_name_n] of the key path. + + Raises: + datastore_errors.BadKeyError if this key does not have a valid path. + """ + + path = [] + for path_element in self.__reference.path().element_list(): + path.append(path_element.type().decode('utf-8')) + if path_element.has_name(): + path.append(path_element.name().decode('utf-8')) + elif path_element.has_id(): + path.append(path_element.id()) + elif _default_id is not None: + path.append(_default_id) + else: + raise datastore_errors.BadKeyError('Incomplete key found in to_path') + return path + @staticmethod def from_path(*args, **kwds): """Static method to construct a Key out of a "path" (kind, id or name, ...). @@ -202,7 +411,10 @@ class Key(object): BadKeyError if the parent key is incomplete. """ parent = kwds.pop('parent', None) - _app = ResolveAppId(kwds.pop('_app', None)) + _app_id_namespace_obj = ResolveAppIdNamespace( + kwds.pop('_app', None), + kwds.pop('_namespace', None), + kwds.pop('_app_id_namespace', None)) if kwds: raise datastore_errors.BadArgumentError( @@ -221,17 +433,18 @@ class Key(object): if not parent.has_id_or_name(): raise datastore_errors.BadKeyError( 'The parent Key is incomplete.') - if _app != parent.app(): + if _app_id_namespace_obj != parent.app_id_namespace(): raise datastore_errors.BadArgumentError( - 'The _app argument (%r) should match parent.app() (%s)' % - (_app, parent.app())) + 'The app_id/namespace arguments (%r) should match ' + + 'parent.app_id_namespace().to_encoded() (%s)' % + (_app_id_namespace_obj, parent.app_id_namespace())) key = Key() ref = key.__reference if parent is not None: ref.CopyFrom(parent.__reference) else: - ref.set_app(_app) + ref.set_app(_app_id_namespace_obj.to_encoded()) path = ref.mutable_path() for i in xrange(0, len(args), 2): @@ -248,9 +461,6 @@ class Key(object): elem.set_id(id_or_name) elif isinstance(id_or_name, basestring): ValidateString(id_or_name, 'name') - if id_or_name and id_or_name[0] in string.digits: - raise datastore_errors.BadArgumentError( - 'Names may not begin with a digit; received %s.' % id_or_name) elem.set_name(id_or_name.encode('utf-8')) else: raise datastore_errors.BadArgumentError( @@ -263,7 +473,21 @@ class Key(object): def app(self): """Returns this entity's app id, a string.""" if self.__reference.app(): - return self.__reference.app().decode('utf-8') + return self.app_id_namespace().app_id().decode('utf-8') + else: + return None + + def namespace(self): + """Returns this entity's app id, a string.""" + if self.__reference.app(): + return self.app_id_namespace().namespace().decode('utf-8') + else: + return None + + def app_id_namespace(self): + """Returns this entity's app id/namespace, an appIdNamespace object.""" + if self.__reference.app(): + return parse_app_id_namespace(self.__reference.app()) else: return None @@ -339,11 +563,13 @@ class Key(object): raise datastore_errors.BadKeyError( 'ToTagUri() called for an entity with an incomplete key.') - return u'tag:%s.%s,%s:%s[%s]' % (saxutils.escape(self.app()), - os.environ['AUTH_DOMAIN'], - datetime.date.today().isoformat(), - saxutils.escape(self.kind()), - saxutils.escape(str(self))) + return u'tag:%s.%s,%s:%s[%s]' % ( + saxutils.escape(self.app_id_namespace().to_encoded()), + os.environ['AUTH_DOMAIN'], + datetime.date.today().isoformat(), + saxutils.escape(self.kind()), + saxutils.escape(str(self))) + ToXml = ToTagUri def entity_group(self): @@ -412,12 +638,15 @@ class Key(object): Returns: string """ - if (self.has_id_or_name()): - encoded = base64.urlsafe_b64encode(self.__reference.Encode()) - return encoded.replace('=', '') - else: - raise datastore_errors.BadKeyError( - 'Cannot string encode an incomplete key!\n%s' % self.__reference) + if self._str is None: + if (self.has_id_or_name()): + encoded = base64.urlsafe_b64encode(self.__reference.Encode()) + self._str = encoded.replace('=', '') + else: + raise datastore_errors.BadKeyError( + 'Cannot string encode an incomplete key!\n%s' % self.__reference) + return self._str + def __repr__(self): """Returns an eval()able string representation of this key. @@ -430,13 +659,13 @@ class Key(object): """ args = [] for elem in self.__reference.path().element_list(): - args.append(repr(elem.type())) + args.append(repr(elem.type().decode('utf-8'))) if elem.has_name(): args.append(repr(elem.name().decode('utf-8'))) else: args.append(repr(elem.id())) - args.append('_app=%r' % self.__reference.app().decode('utf-8')) + args.append('_app_id_namespace=%r' % self.__reference.app().decode('utf-8')) return u'datastore_types.Key.from_path(%s)' % ', '.join(args) def __cmp__(self, other): @@ -456,28 +685,18 @@ class Key(object): if not isinstance(other, Key): return -2 - self_args = [] - other_args = [] + self_args = [self.__reference.app()] + self_args += self.to_path(_default_id=0) - self_args.append(self.__reference.app().decode('utf-8')) - other_args.append(other.__reference.app().decode('utf-8')) + other_args = [other.__reference.app()] + other_args += other.to_path(_default_id=0) - for elem in self.__reference.path().element_list(): - self_args.append(repr(elem.type())) - if elem.has_name(): - self_args.append(repr(elem.name().decode('utf-8'))) - else: - self_args.append(elem.id()) + for self_component, other_component in zip(self_args, other_args): + comparison = cmp(self_component, other_component) + if comparison != 0: + return comparison - for elem in other.__reference.path().element_list(): - other_args.append(repr(elem.type())) - if elem.has_name(): - other_args.append(repr(elem.name().decode('utf-8'))) - else: - other_args.append(elem.id()) - - result = cmp(self_args, other_args) - return result + return cmp(len(self_args), len(other_args)) def __hash__(self): """Returns a 32-bit integer hash of this key. @@ -488,7 +707,9 @@ class Key(object): Returns: int """ - return hash(self.__str__()) + args = self.to_path(_default_id=0) + args.append(self.__reference.app()) + return hash(type(args)) ^ hash(tuple(args)) class Category(unicode): @@ -677,12 +898,12 @@ class IM(object): def __init__(self, protocol, address=None): if address is None: try: - split = protocol.split(' ') + split = protocol.split(' ', 1) protocol, address = split except (AttributeError, ValueError): raise datastore_errors.BadValueError( 'Expected string of format "protocol address"; received %s' % - str(protocol)) + (protocol,)) ValidateString(address, 'address') if protocol not in self.PROTOCOLS: @@ -698,6 +919,7 @@ class IM(object): except datastore_errors.BadValueError: return NotImplemented + return cmp((self.address, self.protocol), (other.address, other.protocol)) @@ -900,6 +1122,63 @@ class ByteString(str): return saxutils.escape(encoded) +class BlobKey(object): + """Key used to identify a blob in Blobstore. + + This object wraps a string that gets used internally by the Blobstore API + to identify application blobs. The BlobKey corresponds to the entity name + of the underlying BlobReference entity. The structure of the key is: + + _<blob-key> + + This class is exposed in the API in both google.appengine.ext.db and + google.appengine.ext.blobstore. + """ + + def __init__(self, blob_key): + """Constructor. + + Used to convert a string to a BlobKey. Normally used internally by + Blobstore API. + + Args: + blob_key: Key name of BlobReference that this key belongs to. + """ + self.__blob_key = blob_key + + def __str__(self): + """Convert to string.""" + return self.__blob_key + + def __repr__(self): + """Returns an eval()able string representation of this key. + + Returns a Python string of the form 'datastore_types.BlobKey(...)' + that can be used to recreate this key. + + Returns: + string + """ + s = type(self).__module__ + return '%s.%s(%r)' % (type(self).__module__, + type(self).__name__, + self.__blob_key) + + def __cmp__(self, other): + if type(other) is type(self): + return cmp(str(self), str(other)) + elif isinstance(other, basestring): + return cmp(self.__blob_key, other) + else: + return NotImplemented + + def __hash__(self): + return hash(self.__blob_key) + + def ToXml(self): + return str(self) + + _PROPERTY_MEANINGS = { @@ -916,6 +1195,7 @@ _PROPERTY_MEANINGS = { PhoneNumber: entity_pb.Property.GD_PHONENUMBER, PostalAddress: entity_pb.Property.GD_POSTALADDRESS, Rating: entity_pb.Property.GD_RATING, + BlobKey: entity_pb.Property.BLOBKEY, } _PROPERTY_TYPES = frozenset([ @@ -940,17 +1220,10 @@ _PROPERTY_TYPES = frozenset([ type(None), unicode, users.User, + BlobKey, ]) -_RAW_PROPERTY_TYPES = frozenset([ - Blob, - Text, -]) - -_STRING_TYPES = frozenset([ - str, - unicode, -]) +_RAW_PROPERTY_TYPES = (Blob, Text) def ValidatePropertyInteger(name, value): """Raises an exception if the supplied integer is invalid. @@ -1051,6 +1324,7 @@ _VALIDATE_PROPERTY_VALUES = { type(None): ValidatePropertyNothing, unicode: ValidatePropertyString, users.User: ValidatePropertyNothing, + BlobKey: ValidatePropertyString, } assert set(_VALIDATE_PROPERTY_VALUES.iterkeys()) == _PROPERTY_TYPES @@ -1143,7 +1417,7 @@ def PackDatetime(name, value, pbvalue): def DatetimeToTimestamp(value): - """Converts a datetime.datetime to seconds since the epoch, as a float. + """Converts a datetime.datetime to microseconds since the epoch, as a float. Args: value: datetime.datetime @@ -1179,6 +1453,10 @@ def PackUser(name, value, pbvalue): value.auth_domain().encode('utf-8')) pbvalue.mutable_uservalue().set_gaiaid(0) + if value.user_id() is not None: + pbvalue.mutable_uservalue().set_obfuscated_gaiaid( + value.user_id().encode('utf-8')) + def PackKey(name, value, pbvalue): """Packs a reference property into a entity_pb.PropertyValue. @@ -1226,6 +1504,7 @@ def PackFloat(name, value, pbvalue): """ pbvalue.set_doublevalue(value) + _PACK_PROPERTY_VALUES = { Blob: PackBlob, ByteString: PackBlob, @@ -1248,6 +1527,7 @@ _PACK_PROPERTY_VALUES = { type(None): lambda name, value, pbvalue: None, unicode: PackString, users.User: PackUser, + BlobKey: PackString, } assert set(_PACK_PROPERTY_VALUES.iterkeys()) == _PROPERTY_TYPES @@ -1335,7 +1615,6 @@ _PROPERTY_CONVERSIONS = { entity_pb.Property.ATOM_CATEGORY: Category, entity_pb.Property.ATOM_LINK: Link, entity_pb.Property.GD_EMAIL: Email, - entity_pb.Property.GEORSS_POINT: lambda coords: GeoPt(*coords), entity_pb.Property.GD_IM: IM, entity_pb.Property.GD_PHONENUMBER: PhoneNumber, entity_pb.Property.GD_POSTALADDRESS: PostalAddress, @@ -1343,6 +1622,7 @@ _PROPERTY_CONVERSIONS = { entity_pb.Property.BLOB: Blob, entity_pb.Property.BYTESTRING: ByteString, entity_pb.Property.TEXT: Text, + entity_pb.Property.BLOBKEY: BlobKey, } @@ -1372,16 +1652,20 @@ def FromPropertyPb(pb): elif pbval.has_referencevalue(): value = FromReferenceProperty(pbval) elif pbval.has_pointvalue(): - value = (pbval.pointvalue().x(), pbval.pointvalue().y()) + value = GeoPt(pbval.pointvalue().x(), pbval.pointvalue().y()) elif pbval.has_uservalue(): email = unicode(pbval.uservalue().email().decode('utf-8')) auth_domain = unicode(pbval.uservalue().auth_domain().decode('utf-8')) - value = users.User(email=email, _auth_domain=auth_domain) + obfuscated_gaiaid = pbval.uservalue().obfuscated_gaiaid().decode('utf-8') + obfuscated_gaiaid = unicode(obfuscated_gaiaid) + value = users.User(email=email, + _auth_domain=auth_domain, + _user_id=obfuscated_gaiaid) else: value = None try: - if pb.has_meaning(): + if pb.has_meaning() and pb.meaning() in _PROPERTY_CONVERSIONS: conversion = _PROPERTY_CONVERSIONS[meaning] value = conversion(value) except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg: @@ -1437,6 +1721,7 @@ _PROPERTY_TYPE_STRINGS = { 'gd:phonenumber': PhoneNumber, 'gd:postaladdress': PostalAddress, 'gd:rating': Rating, + 'blobkey': BlobKey, } @@ -1452,7 +1737,9 @@ def FromPropertyTypeName(type_name): return _PROPERTY_TYPE_STRINGS[type_name] -def PropertyValueFromString(type_, value_string, _auth_domain=None): +def PropertyValueFromString(type_, + value_string, + _auth_domain=None): """Returns an instance of a property value given a type and string value. The reverse of this method is just str() and type() of the python value. google_appengine/google/appengine/api/datastore_types.py @@ -42,6 +42,24 @@ PNG = images_service_pb.OutputSettings.PNG OUTPUT_ENCODING_TYPES = frozenset([JPEG, PNG]) +TOP_LEFT = images_service_pb.CompositeImageOptions.TOP_LEFT +TOP_CENTER = images_service_pb.CompositeImageOptions.TOP +TOP_RIGHT = images_service_pb.CompositeImageOptions.TOP_RIGHT +CENTER_LEFT = images_service_pb.CompositeImageOptions.LEFT +CENTER_CENTER = images_service_pb.CompositeImageOptions.CENTER +CENTER_RIGHT = images_service_pb.CompositeImageOptions.RIGHT +BOTTOM_LEFT = images_service_pb.CompositeImageOptions.BOTTOM_LEFT +BOTTOM_CENTER = images_service_pb.CompositeImageOptions.BOTTOM +BOTTOM_RIGHT = images_service_pb.CompositeImageOptions.BOTTOM_RIGHT + +ANCHOR_TYPES = frozenset([TOP_LEFT, TOP_CENTER, TOP_RIGHT, CENTER_LEFT, + CENTER_CENTER, CENTER_RIGHT, BOTTOM_LEFT, + BOTTOM_CENTER, BOTTOM_RIGHT]) + +MAX_TRANSFORMS_PER_REQUEST = 10 + +MAX_COMPOSITES_PER_REQUEST = 16 + class Error(Exception): """Base error class for this module.""" @@ -67,46 +85,44 @@ class LargeImageError(Error): """The image data given is too large to process.""" +class InvalidBlobKeyError(Error): + """The provided blob key was invalid.""" + + class Image(object): """Image object to manipulate.""" - def __init__(self, image_data): + def __init__(self, image_data=None, blob_key=None): """Constructor. Args: image_data: str, image data in string form. + blob_key: str, image data as a blobstore blob key. Raises: NotImageError if the given data is empty. """ - if not image_data: + if not image_data and not blob_key: raise NotImageError("Empty image data.") + if image_data and blob_key: + raise NotImageError("Can only take one image or blob key.") self._image_data = image_data + self._blob_key = blob_key self._transforms = [] - self._transform_map = {} self._width = None self._height = None - def _check_transform_limits(self, transform): + def _check_transform_limits(self): """Ensure some simple limits on the number of transforms allowed. - Args: - transform: images_service_pb.ImagesServiceTransform, enum of the - trasnform called. - Raises: - BadRequestError if the transform has already been requested for the image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested for this image """ - if not images_service_pb.ImagesServiceTransform.Type_Name(transform): - raise BadRequestError("'%s' is not a valid transform." % transform) - - if transform in self._transform_map: - transform_name = images_service_pb.ImagesServiceTransform.Type_Name( - transform) - raise BadRequestError("A '%s' transform has already been " - "requested on this image." % transform_name) - self._transform_map[transform] = True + if len(self._transforms) >= MAX_TRANSFORMS_PER_REQUEST: + raise BadRequestError("%d transforms have already been requested on this " + "image." % MAX_TRANSFORMS_PER_REQUEST) def _update_dimensions(self): """Updates the width and height fields of the image. @@ -115,6 +131,8 @@ class Image(object): NotImageError if the image data is not an image. BadImageError if the image data is corrupt. """ + if not self._image_data: + raise NotImageError("Dimensions unavailable for blob key input") size = len(self._image_data) if size >= 6 and self._image_data.startswith("GIF"): self._update_gif_dimensions() @@ -172,14 +190,14 @@ class Image(object): if (offset < size and ord(self._image_data[offset]) & 0xF0 == 0xC0 and ord(self._image_data[offset]) != 0xC4): offset += 4 - if offset + 4 < size: + if offset + 4 <= size: self._height, self._width = struct.unpack( ">HH", self._image_data[offset:offset + 4]) break else: raise BadImageError("Corrupt JPEG format") - elif offset + 2 <= size: + elif offset + 3 <= size: offset += 1 offset += struct.unpack(">H", self._image_data[offset:offset + 2])[0] else: @@ -199,7 +217,7 @@ class Image(object): else: endianness = ">" ifd_offset = struct.unpack(endianness + "I", self._image_data[4:8])[0] - if ifd_offset < size + 14: + if ifd_offset + 14 <= size: ifd_size = struct.unpack( endianness + "H", self._image_data[ifd_offset:ifd_offset + 2])[0] @@ -291,7 +309,8 @@ class Image(object): Raises: TypeError when width or height is not either 'int' or 'long' types. BadRequestError when there is something wrong with the given height or - width or if a Resize has already been requested on this image. + width or if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested on this image. """ if (not isinstance(width, (int, long)) or not isinstance(height, (int, long))): @@ -303,10 +322,9 @@ class Image(object): raise BadRequestError("At least one of width or height must be > 0.") if width > 4000 or height > 4000: - raise BadRequestError("Both width and height must be < 4000.") + raise BadRequestError("Both width and height must be <= 4000.") - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.RESIZE) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_width(width) @@ -323,7 +341,7 @@ class Image(object): Raises: TypeError when degrees is not either 'int' or 'long' types. BadRequestError when there is something wrong with the given degrees or - if a Rotate trasnform has already been requested. + if MAX_TRANSFORMS_PER_REQUEST transforms have already been requested. """ if not isinstance(degrees, (int, long)): raise TypeError("Degrees must be integers.") @@ -333,8 +351,7 @@ class Image(object): degrees = degrees % 360 - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.ROTATE) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_rotate(degrees) @@ -345,11 +362,10 @@ class Image(object): """Flip the image horizontally. Raises: - BadRequestError if a HorizontalFlip has already been requested on the - image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested on the image. """ - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.HORIZONTAL_FLIP) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_horizontal_flip(True) @@ -360,11 +376,10 @@ class Image(object): """Flip the image vertically. Raises: - BadRequestError if a HorizontalFlip has already been requested on the - image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already been + requested on the image. """ - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.VERTICAL_FLIP) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_vertical_flip(True) @@ -405,7 +420,8 @@ class Image(object): Raises: TypeError if the args are not of type 'float'. BadRequestError when there is something wrong with the given bounding box - or if there has already been a crop transform requested for this image. + or if MAX_TRANSFORMS_PER_REQUEST transforms have already been requested + for this image. """ self._validate_crop_arg(left_x, "left_x") self._validate_crop_arg(top_y, "top_y") @@ -417,7 +433,7 @@ class Image(object): if top_y >= bottom_y: raise BadRequestError("top_y must be less than bottom_y") - self._check_transform_limits(images_service_pb.ImagesServiceTransform.CROP) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_crop_left_x(left_x) @@ -433,16 +449,27 @@ class Image(object): This is similar to the "I'm Feeling Lucky" button in Picasa. Raises: - BadRequestError if this transform has already been requested for this - image. + BadRequestError if MAX_TRANSFORMS_PER_REQUEST transforms have already + been requested for this image. """ - self._check_transform_limits( - images_service_pb.ImagesServiceTransform.IM_FEELING_LUCKY) + self._check_transform_limits() transform = images_service_pb.Transform() transform.set_autolevels(True) self._transforms.append(transform) + def _set_imagedata(self, imagedata): + """Fills in an ImageData PB from this Image instance. + + Args: + imagedata: An ImageData PB instance + """ + if self._blob_key: + imagedata.set_content("") + imagedata.set_blob_key(self._blob_key) + else: + imagedata.set_content(self._image_data) + def execute_transforms(self, output_encoding=PNG): """Perform transformations on given image. @@ -458,6 +485,7 @@ class Image(object): NotImageError when the image data given is not an image. BadImageError when the image data given is corrupt. LargeImageError when the image data given is too large to process. + InvalidBlobKeyError when the blob key provided is invalid. TransformtionError when something errors during image manipulation. Error when something unknown, but bad, happens. """ @@ -471,7 +499,7 @@ class Image(object): request = images_service_pb.ImagesTransformRequest() response = images_service_pb.ImagesTransformResponse() - request.mutable_image().set_content(self._image_data) + self._set_imagedata(request.mutable_image()) for transform in self._transforms: request.add_transform().CopyFrom(transform) @@ -497,14 +525,17 @@ class Image(object): images_service_pb.ImagesServiceError.IMAGE_TOO_LARGE): raise LargeImageError() elif (e.application_error == + images_service_pb.ImagesServiceError.INVALID_BLOB_KEY): + raise InvalidBlobKeyError() + elif (e.application_error == images_service_pb.ImagesServiceError.UNSPECIFIED_ERROR): raise TransformationError() else: raise Error() self._image_data = response.image().content() + self._blob_key = None self._transforms = [] - self._transform_map.clear() self._width = None self._height = None return self._image_data @@ -523,6 +554,51 @@ class Image(object): self._update_dimensions() return self._height + def histogram(self): + """Calculates the histogram of the image. + + Returns: 3 256-element lists containing the number of occurences of each + value of each color in the order RGB. As described at + http://en.wikipedia.org/wiki/Color_histogram for N = 256. i.e. the first + value of the first list contains the number of pixels with a red value of + 0, the second the number with a red value of 1. + + Raises: + NotImageError when the image data given is not an image. + BadImageError when the image data given is corrupt. + LargeImageError when the image data given is too large to process. + Error when something unknown, but bad, happens. + """ + request = images_service_pb.ImagesHistogramRequest() + response = images_service_pb.ImagesHistogramResponse() + + self._set_imagedata(request.mutable_image()) + + try: + apiproxy_stub_map.MakeSyncCall("images", + "Histogram", + request, + response) + except apiproxy_errors.ApplicationError, e: + if (e.application_error == + images_service_pb.ImagesServiceError.NOT_IMAGE): + raise NotImageError() + elif (e.application_error == + images_service_pb.ImagesServiceError.BAD_IMAGE_DATA): + raise BadImageError() + elif (e.application_error == + images_service_pb.ImagesServiceError.IMAGE_TOO_LARGE): + raise LargeImageError() + elif (e.application_error == + images_service_pb.ImagesServiceError.INVALID_BLOB_KEY): + raise InvalidBlobKeyError() + else: + raise Error() + histogram = response.histogram() + return [histogram.red_list(), + histogram.green_list(), + histogram.blue_list()] + def resize(image_data, width=0, height=0, output_encoding=PNG): """Resize a given image file maintaining the aspect ratio. @@ -540,7 +616,7 @@ def resize(image_data, width=0, height=0, output_encoding=PNG): Raises: TypeError when width or height not either 'int' or 'long' types. BadRequestError when there is something wrong with the given height or - width or if a Resize has already been requested on this image. + width. Error when something went wrong with the call. See Image.ExecuteTransforms for more details. """ @@ -559,8 +635,7 @@ def rotate(image_data, degrees, output_encoding=PNG): Raises: TypeError when degrees is not either 'int' or 'long' types. - BadRequestError when there is something wrong with the given degrees or - if a Rotate trasnform has already been requested. + BadRequestError when there is something wrong with the given degrees. Error when something went wrong with the call. See Image.ExecuteTransforms for more details. """ @@ -619,8 +694,7 @@ def crop(image_data, left_x, top_y, right_x, bottom_y, output_encoding=PNG): Raises: TypeError if the args are not of type 'float'. - BadRequestError when there is something wrong with the given bounding box - or if there has already been a crop transform requested for this image. + BadRequestError when there is something wrong with the given bounding box. Error when something went wrong with the call. See Image.ExecuteTransforms for more details. """ @@ -645,3 +719,147 @@ def im_feeling_lucky(image_data, output_encoding=PNG): image = Image(image_data) image.im_feeling_lucky() return image.execute_transforms(output_encoding=output_encoding) + +def composite(inputs, width, height, color=0, output_encoding=PNG): + """Composite one or more images onto a canvas. + + Args: + inputs: a list of tuples (image_data, x_offset, y_offset, opacity, anchor) + where + image_data: str, source image data. + x_offset: x offset in pixels from the anchor position + y_offset: y offset in piyels from the anchor position + opacity: opacity of the image specified as a float in range [0.0, 1.0] + anchor: anchoring point from ANCHOR_POINTS. The anchor point of the image + is aligned with the same anchor point of the canvas. e.g. TOP_RIGHT would + place the top right corner of the image at the top right corner of the + canvas then apply the x and y offsets. + width: canvas width in pixels. + height: canvas height in pixels. + color: canvas background color encoded as a 32 bit unsigned int where each + color channel is represented by one byte in order ARGB. + output_encoding: a value from OUTPUT_ENCODING_TYPES. + + Returns: + str, image data of the composited image. + + Raises: + TypeError If width, height, color, x_offset or y_offset are not of type + int or long or if opacity is not a float + BadRequestError If more than MAX_TRANSFORMS_PER_REQUEST compositions have + been requested, if the canvas width or height is greater than 4000 or less + than or equal to 0, if the color is invalid or if for any composition + option, the opacity is outside the range [0,1] or the anchor is invalid. + """ + if (not isinstance(width, (int, long)) or + not isinstance(height, (int, long)) or + not isinstance(color, (int, long))): + raise TypeError("Width, height and color must be integers.") + if output_encoding not in OUTPUT_ENCODING_TYPES: + raise BadRequestError("Output encoding type '%s' not in recognized set " + "%s" % (output_encoding, OUTPUT_ENCODING_TYPES)) + + if not inputs: + raise BadRequestError("Must provide at least one input") + if len(inputs) > MAX_COMPOSITES_PER_REQUEST: + raise BadRequestError("A maximum of %d composition operations can be" + "performed in a single request" % + MAX_COMPOSITES_PER_REQUEST) + + if width <= 0 or height <= 0: + raise BadRequestError("Width and height must be > 0.") + if width > 4000 or height > 4000: + raise BadRequestError("Width and height must be <= 4000.") + + if color > 0xffffffff or color < 0: + raise BadRequestError("Invalid color") + if color >= 0x80000000: + color -= 0x100000000 + + image_map = {} + + request = images_service_pb.ImagesCompositeRequest() + response = images_service_pb.ImagesTransformResponse() + for (image, x, y, opacity, anchor) in inputs: + if not image: + raise BadRequestError("Each input must include an image") + if (not isinstance(x, (int, long)) or + not isinstance(y, (int, long)) or + not isinstance(opacity, (float))): + raise TypeError("x_offset, y_offset must be integers and opacity must" + "be a float") + if x > 4000 or x < -4000: + raise BadRequestError("xOffsets must be in range [-4000, 4000]") + if y > 4000 or y < -4000: + raise BadRequestError("yOffsets must be in range [-4000, 4000]") + if opacity < 0 or opacity > 1: + raise BadRequestError("Opacity must be in the range 0.0 to 1.0") + if anchor not in ANCHOR_TYPES: + raise BadRequestError("Anchor type '%s' not in recognized set %s" % + (anchor, ANCHOR_TYPES)) + if image not in image_map: + image_map[image] = request.image_size() + + if isinstance(image, Image): + image._set_imagedata(request.add_image()) + else: + request.add_image().set_content(image) + + option = request.add_options() + option.set_x_offset(x) + option.set_y_offset(y) + option.set_opacity(opacity) + option.set_anchor(anchor) + option.set_source_index(image_map[image]) + + request.mutable_canvas().mutable_output().set_mime_type(output_encoding) + request.mutable_canvas().set_width(width) + request.mutable_canvas().set_height(height) + request.mutable_canvas().set_color(color) + + try: + apiproxy_stub_map.MakeSyncCall("images", + "Composite", + request, + response) + except apiproxy_errors.ApplicationError, e: + if (e.application_error == + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA): + raise BadRequestError() + elif (e.application_error == + images_service_pb.ImagesServiceError.NOT_IMAGE): + raise NotImageError() + elif (e.application_error == + images_service_pb.ImagesServiceError.BAD_IMAGE_DATA): + raise BadImageError() + elif (e.application_error == + images_service_pb.ImagesServiceError.IMAGE_TOO_LARGE): + raise LargeImageError() + elif (e.application_error == + images_service_pb.ImagesServiceError.INVALID_BLOB_KEY): + raise InvalidBlobKeyError() + elif (e.application_error == + images_service_pb.ImagesServiceError.UNSPECIFIED_ERROR): + raise TransformationError() + else: + raise Error() + + return response.image().content() + + +def histogram(image_data): + """Calculates the histogram of the given image. + + Args: + image_data: str, source image data. + Returns: 3 256-element lists containing the number of occurences of each + value of each color in the order RGB. + + Raises: + NotImageError when the image data given is not an image. + BadImageError when the image data given is corrupt. + LargeImageError when the image data given is too large to process. + Error when something unknown, but bad, happens. + """ + image = Image(image_data) + return image.histogram() google_appengine/google/appengine/api/images/__init__.py @@ -29,6 +29,7 @@ class ImagesServiceError(ProtocolBuffer.ProtocolMessage): NOT_IMAGE = 3 BAD_IMAGE_DATA = 4 IMAGE_TOO_LARGE = 5 + INVALID_BLOB_KEY = 6 _ErrorCode_NAMES = { 1: "UNSPECIFIED_ERROR", @@ -36,6 +37,7 @@ class ImagesServiceError(ProtocolBuffer.ProtocolMessage): 3: "NOT_IMAGE", 4: "BAD_IMAGE_DATA", 5: "IMAGE_TOO_LARGE", + 6: "INVALID_BLOB_KEY", } def ErrorCode_Name(cls, x): return cls._ErrorCode_NAMES.get(x, "") @@ -80,13 +82,17 @@ class ImagesServiceError(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -150,13 +156,17 @@ class ImagesServiceTransform(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -192,8 +202,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.width_ = x def clear_width(self): - self.has_width_ = 0 - self.width_ = 0 + if self.has_width_: + self.has_width_ = 0 + self.width_ = 0 def has_width(self): return self.has_width_ @@ -204,8 +215,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.height_ = x def clear_height(self): - self.has_height_ = 0 - self.height_ = 0 + if self.has_height_: + self.has_height_ = 0 + self.height_ = 0 def has_height(self): return self.has_height_ @@ -216,8 +228,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.rotate_ = x def clear_rotate(self): - self.has_rotate_ = 0 - self.rotate_ = 0 + if self.has_rotate_: + self.has_rotate_ = 0 + self.rotate_ = 0 def has_rotate(self): return self.has_rotate_ @@ -228,8 +241,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.horizontal_flip_ = x def clear_horizontal_flip(self): - self.has_horizontal_flip_ = 0 - self.horizontal_flip_ = 0 + if self.has_horizontal_flip_: + self.has_horizontal_flip_ = 0 + self.horizontal_flip_ = 0 def has_horizontal_flip(self): return self.has_horizontal_flip_ @@ -240,8 +254,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.vertical_flip_ = x def clear_vertical_flip(self): - self.has_vertical_flip_ = 0 - self.vertical_flip_ = 0 + if self.has_vertical_flip_: + self.has_vertical_flip_ = 0 + self.vertical_flip_ = 0 def has_vertical_flip(self): return self.has_vertical_flip_ @@ -252,8 +267,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.crop_left_x_ = x def clear_crop_left_x(self): - self.has_crop_left_x_ = 0 - self.crop_left_x_ = 0.0 + if self.has_crop_left_x_: + self.has_crop_left_x_ = 0 + self.crop_left_x_ = 0.0 def has_crop_left_x(self): return self.has_crop_left_x_ @@ -264,8 +280,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.crop_top_y_ = x def clear_crop_top_y(self): - self.has_crop_top_y_ = 0 - self.crop_top_y_ = 0.0 + if self.has_crop_top_y_: + self.has_crop_top_y_ = 0 + self.crop_top_y_ = 0.0 def has_crop_top_y(self): return self.has_crop_top_y_ @@ -276,8 +293,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.crop_right_x_ = x def clear_crop_right_x(self): - self.has_crop_right_x_ = 0 - self.crop_right_x_ = 1.0 + if self.has_crop_right_x_: + self.has_crop_right_x_ = 0 + self.crop_right_x_ = 1.0 def has_crop_right_x(self): return self.has_crop_right_x_ @@ -288,8 +306,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.crop_bottom_y_ = x def clear_crop_bottom_y(self): - self.has_crop_bottom_y_ = 0 - self.crop_bottom_y_ = 1.0 + if self.has_crop_bottom_y_: + self.has_crop_bottom_y_ = 0 + self.crop_bottom_y_ = 1.0 def has_crop_bottom_y(self): return self.has_crop_bottom_y_ @@ -300,8 +319,9 @@ class Transform(ProtocolBuffer.ProtocolMessage): self.autolevels_ = x def clear_autolevels(self): - self.has_autolevels_ = 0 - self.autolevels_ = 0 + if self.has_autolevels_: + self.has_autolevels_ = 0 + self.autolevels_ = 0 def has_autolevels(self): return self.has_autolevels_ @@ -456,6 +476,10 @@ class Transform(ProtocolBuffer.ProtocolMessage): if self.has_autolevels_: res+=prefix+("autolevels: %s\n" % self.DebugFormatBool(self.autolevels_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kwidth = 1 kheight = 2 krotate = 3 @@ -467,49 +491,41 @@ class Transform(ProtocolBuffer.ProtocolMessage): kcrop_bottom_y = 9 kautolevels = 10 - _TEXT = ( - "ErrorCode", - "width", - "height", - "rotate", - "horizontal_flip", - "vertical_flip", - "crop_left_x", - "crop_top_y", - "crop_right_x", - "crop_bottom_y", - "autolevels", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.FLOAT, - - ProtocolBuffer.Encoder.FLOAT, - - ProtocolBuffer.Encoder.FLOAT, - - ProtocolBuffer.Encoder.FLOAT, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "width", + 2: "height", + 3: "rotate", + 4: "horizontal_flip", + 5: "vertical_flip", + 6: "crop_left_x", + 7: "crop_top_y", + 8: "crop_right_x", + 9: "crop_bottom_y", + 10: "autolevels", + }, 10) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.NUMERIC, + 6: ProtocolBuffer.Encoder.FLOAT, + 7: ProtocolBuffer.Encoder.FLOAT, + 8: ProtocolBuffer.Encoder.FLOAT, + 9: ProtocolBuffer.Encoder.FLOAT, + 10: ProtocolBuffer.Encoder.NUMERIC, + }, 10, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" class ImageData(ProtocolBuffer.ProtocolMessage): has_content_ = 0 content_ = "" + has_blob_key_ = 0 + blob_key_ = "" def __init__(self, contents=None): if contents is not None: self.MergeFromString(contents) @@ -521,20 +537,37 @@ class ImageData(ProtocolBuffer.ProtocolMessage): self.content_ = x def clear_content(self): - self.has_content_ = 0 - self.content_ = "" + if self.has_content_: + self.has_content_ = 0 + self.content_ = "" def has_content(self): return self.has_content_ + def blob_key(self): return self.blob_key_ + + def set_blob_key(self, x): + self.has_blob_key_ = 1 + self.blob_key_ = x + + def clear_blob_key(self): + if self.has_blob_key_: + self.has_blob_key_ = 0 + self.blob_key_ = "" + + def has_blob_key(self): return self.has_blob_key_ + def MergeFrom(self, x): assert x is not self if (x.has_content()): self.set_content(x.content()) + if (x.has_blob_key()): self.set_blob_key(x.blob_key()) def Equals(self, x): if x is self: return 1 if self.has_content_ != x.has_content_: return 0 if self.has_content_ and self.content_ != x.content_: return 0 + if self.has_blob_key_ != x.has_blob_key_: return 0 + if self.has_blob_key_ and self.blob_key_ != x.blob_key_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -548,14 +581,19 @@ class ImageData(ProtocolBuffer.ProtocolMessage): def ByteSize(self): n = 0 n += self.lengthString(len(self.content_)) + if (self.has_blob_key_): n += 1 + self.lengthString(len(self.blob_key_)) return n + 1 def Clear(self): self.clear_content() + self.clear_blob_key() def OutputUnchecked(self, out): out.putVarInt32(10) out.putPrefixedString(self.content_) + if (self.has_blob_key_): + out.putVarInt32(18) + out.putPrefixedString(self.blob_key_) def TryMerge(self, d): while d.avail() > 0: @@ -563,6 +601,9 @@ class ImageData(ProtocolBuffer.ProtocolMessage): if tt == 10: self.set_content(d.getPrefixedString()) continue + if tt == 18: + self.set_blob_key(d.getPrefixedString()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -570,20 +611,27 @@ class ImageData(ProtocolBuffer.ProtocolMessage): def __str__(self, prefix="", printElemNumber=0): res="" if self.has_content_: res+=prefix+("content: %s\n" % self.DebugFormatString(self.content_)) + if self.has_blob_key_: res+=prefix+("blob_key: %s\n" % self.DebugFormatString(self.blob_key_)) return res - kcontent = 1 - _TEXT = ( - "ErrorCode", - "content", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + kcontent = 1 + kblob_key = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "content", + 2: "blob_key", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -613,8 +661,9 @@ class OutputSettings(ProtocolBuffer.ProtocolMessage): self.mime_type_ = x def clear_mime_type(self): - self.has_mime_type_ = 0 - self.mime_type_ = 0 + if self.has_mime_type_: + self.has_mime_type_ = 0 + self.mime_type_ = 0 def has_mime_type(self): return self.has_mime_type_ @@ -661,18 +710,21 @@ class OutputSettings(ProtocolBuffer.ProtocolMessage): if self.has_mime_type_: res+=prefix+("mime_type: %s\n" % self.DebugFormatInt32(self.mime_type_)) return res - kmime_type = 1 - _TEXT = ( - "ErrorCode", - "mime_type", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kmime_type = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "mime_type", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -822,30 +874,723 @@ class ImagesTransformRequest(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kimage = 1 ktransform = 2 koutput = 3 - _TEXT = ( - "ErrorCode", - "image", - "transform", - "output", - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "image", + 2: "transform", + 3: "output", + }, 3) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + }, 3, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ImagesTransformResponse(ProtocolBuffer.ProtocolMessage): + has_image_ = 0 + + def __init__(self, contents=None): + self.image_ = ImageData() + if contents is not None: self.MergeFromString(contents) + + def image(self): return self.image_ - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + def mutable_image(self): self.has_image_ = 1; return self.image_ + + def clear_image(self):self.has_image_ = 0; self.image_.Clear() + + def has_image(self): return self.has_image_ - ProtocolBuffer.Encoder.STRING, - ProtocolBuffer.Encoder.STRING, + def MergeFrom(self, x): + assert x is not self + if (x.has_image()): self.mutable_image().MergeFrom(x.image()) - ) + def Equals(self, x): + if x is self: return 1 + if self.has_image_ != x.has_image_: return 0 + if self.has_image_ and self.image_ != x.image_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_image_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: image not set.') + elif not self.image_.IsInitialized(debug_strs): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(self.image_.ByteSize()) + return n + 1 + + def Clear(self): + self.clear_image() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putVarInt32(self.image_.ByteSize()) + self.image_.OutputUnchecked(out) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_image().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_image_: + res+=prefix+"image <\n" + res+=self.image_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kimage = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "image", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" -class ImagesTransformResponse(ProtocolBuffer.ProtocolMessage): +class CompositeImageOptions(ProtocolBuffer.ProtocolMessage): + + TOP_LEFT = 0 + TOP = 1 + TOP_RIGHT = 2 + LEFT = 3 + CENTER = 4 + RIGHT = 5 + BOTTOM_LEFT = 6 + BOTTOM = 7 + BOTTOM_RIGHT = 8 + + _ANCHOR_NAMES = { + 0: "TOP_LEFT", + 1: "TOP", + 2: "TOP_RIGHT", + 3: "LEFT", + 4: "CENTER", + 5: "RIGHT", + 6: "BOTTOM_LEFT", + 7: "BOTTOM", + 8: "BOTTOM_RIGHT", + } + + def ANCHOR_Name(cls, x): return cls._ANCHOR_NAMES.get(x, "") + ANCHOR_Name = classmethod(ANCHOR_Name) + + has_source_index_ = 0 + source_index_ = 0 + has_x_offset_ = 0 + x_offset_ = 0 + has_y_offset_ = 0 + y_offset_ = 0 + has_opacity_ = 0 + opacity_ = 0.0 + has_anchor_ = 0 + anchor_ = 0 + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def source_index(self): return self.source_index_ + + def set_source_index(self, x): + self.has_source_index_ = 1 + self.source_index_ = x + + def clear_source_index(self): + if self.has_source_index_: + self.has_source_index_ = 0 + self.source_index_ = 0 + + def has_source_index(self): return self.has_source_index_ + + def x_offset(self): return self.x_offset_ + + def set_x_offset(self, x): + self.has_x_offset_ = 1 + self.x_offset_ = x + + def clear_x_offset(self): + if self.has_x_offset_: + self.has_x_offset_ = 0 + self.x_offset_ = 0 + + def has_x_offset(self): return self.has_x_offset_ + + def y_offset(self): return self.y_offset_ + + def set_y_offset(self, x): + self.has_y_offset_ = 1 + self.y_offset_ = x + + def clear_y_offset(self): + if self.has_y_offset_: + self.has_y_offset_ = 0 + self.y_offset_ = 0 + + def has_y_offset(self): return self.has_y_offset_ + + def opacity(self): return self.opacity_ + + def set_opacity(self, x): + self.has_opacity_ = 1 + self.opacity_ = x + + def clear_opacity(self): + if self.has_opacity_: + self.has_opacity_ = 0 + self.opacity_ = 0.0 + + def has_opacity(self): return self.has_opacity_ + + def anchor(self): return self.anchor_ + + def set_anchor(self, x): + self.has_anchor_ = 1 + self.anchor_ = x + + def clear_anchor(self): + if self.has_anchor_: + self.has_anchor_ = 0 + self.anchor_ = 0 + + def has_anchor(self): return self.has_anchor_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_source_index()): self.set_source_index(x.source_index()) + if (x.has_x_offset()): self.set_x_offset(x.x_offset()) + if (x.has_y_offset()): self.set_y_offset(x.y_offset()) + if (x.has_opacity()): self.set_opacity(x.opacity()) + if (x.has_anchor()): self.set_anchor(x.anchor()) + + def Equals(self, x): + if x is self: return 1 + if self.has_source_index_ != x.has_source_index_: return 0 + if self.has_source_index_ and self.source_index_ != x.source_index_: return 0 + if self.has_x_offset_ != x.has_x_offset_: return 0 + if self.has_x_offset_ and self.x_offset_ != x.x_offset_: return 0 + if self.has_y_offset_ != x.has_y_offset_: return 0 + if self.has_y_offset_ and self.y_offset_ != x.y_offset_: return 0 + if self.has_opacity_ != x.has_opacity_: return 0 + if self.has_opacity_ and self.opacity_ != x.opacity_: return 0 + if self.has_anchor_ != x.has_anchor_: return 0 + if self.has_anchor_ and self.anchor_ != x.anchor_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_source_index_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: source_index not set.') + if (not self.has_x_offset_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: x_offset not set.') + if (not self.has_y_offset_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: y_offset not set.') + if (not self.has_opacity_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: opacity not set.') + if (not self.has_anchor_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: anchor not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthVarInt64(self.source_index_) + n += self.lengthVarInt64(self.x_offset_) + n += self.lengthVarInt64(self.y_offset_) + n += self.lengthVarInt64(self.anchor_) + return n + 9 + + def Clear(self): + self.clear_source_index() + self.clear_x_offset() + self.clear_y_offset() + self.clear_opacity() + self.clear_anchor() + + def OutputUnchecked(self, out): + out.putVarInt32(8) + out.putVarInt32(self.source_index_) + out.putVarInt32(16) + out.putVarInt32(self.x_offset_) + out.putVarInt32(24) + out.putVarInt32(self.y_offset_) + out.putVarInt32(37) + out.putFloat(self.opacity_) + out.putVarInt32(40) + out.putVarInt32(self.anchor_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 8: + self.set_source_index(d.getVarInt32()) + continue + if tt == 16: + self.set_x_offset(d.getVarInt32()) + continue + if tt == 24: + self.set_y_offset(d.getVarInt32()) + continue + if tt == 37: + self.set_opacity(d.getFloat()) + continue + if tt == 40: + self.set_anchor(d.getVarInt32()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_source_index_: res+=prefix+("source_index: %s\n" % self.DebugFormatInt32(self.source_index_)) + if self.has_x_offset_: res+=prefix+("x_offset: %s\n" % self.DebugFormatInt32(self.x_offset_)) + if self.has_y_offset_: res+=prefix+("y_offset: %s\n" % self.DebugFormatInt32(self.y_offset_)) + if self.has_opacity_: res+=prefix+("opacity: %s\n" % self.DebugFormatFloat(self.opacity_)) + if self.has_anchor_: res+=prefix+("anchor: %s\n" % self.DebugFormatInt32(self.anchor_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + ksource_index = 1 + kx_offset = 2 + ky_offset = 3 + kopacity = 4 + kanchor = 5 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "source_index", + 2: "x_offset", + 3: "y_offset", + 4: "opacity", + 5: "anchor", + }, 5) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.FLOAT, + 5: ProtocolBuffer.Encoder.NUMERIC, + }, 5, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ImagesCanvas(ProtocolBuffer.ProtocolMessage): + has_width_ = 0 + width_ = 0 + has_height_ = 0 + height_ = 0 + has_output_ = 0 + has_color_ = 0 + color_ = -1 + + def __init__(self, contents=None): + self.output_ = OutputSettings() + if contents is not None: self.MergeFromString(contents) + + def width(self): return self.width_ + + def set_width(self, x): + self.has_width_ = 1 + self.width_ = x + + def clear_width(self): + if self.has_width_: + self.has_width_ = 0 + self.width_ = 0 + + def has_width(self): return self.has_width_ + + def height(self): return self.height_ + + def set_height(self, x): + self.has_height_ = 1 + self.height_ = x + + def clear_height(self): + if self.has_height_: + self.has_height_ = 0 + self.height_ = 0 + + def has_height(self): return self.has_height_ + + def output(self): return self.output_ + + def mutable_output(self): self.has_output_ = 1; return self.output_ + + def clear_output(self):self.has_output_ = 0; self.output_.Clear() + + def has_output(self): return self.has_output_ + + def color(self): return self.color_ + + def set_color(self, x): + self.has_color_ = 1 + self.color_ = x + + def clear_color(self): + if self.has_color_: + self.has_color_ = 0 + self.color_ = -1 + + def has_color(self): return self.has_color_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_width()): self.set_width(x.width()) + if (x.has_height()): self.set_height(x.height()) + if (x.has_output()): self.mutable_output().MergeFrom(x.output()) + if (x.has_color()): self.set_color(x.color()) + + def Equals(self, x): + if x is self: return 1 + if self.has_width_ != x.has_width_: return 0 + if self.has_width_ and self.width_ != x.width_: return 0 + if self.has_height_ != x.has_height_: return 0 + if self.has_height_ and self.height_ != x.height_: return 0 + if self.has_output_ != x.has_output_: return 0 + if self.has_output_ and self.output_ != x.output_: return 0 + if self.has_color_ != x.has_color_: return 0 + if self.has_color_ and self.color_ != x.color_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_width_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: width not set.') + if (not self.has_height_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: height not set.') + if (not self.has_output_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: output not set.') + elif not self.output_.IsInitialized(debug_strs): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthVarInt64(self.width_) + n += self.lengthVarInt64(self.height_) + n += self.lengthString(self.output_.ByteSize()) + if (self.has_color_): n += 1 + self.lengthVarInt64(self.color_) + return n + 3 + + def Clear(self): + self.clear_width() + self.clear_height() + self.clear_output() + self.clear_color() + + def OutputUnchecked(self, out): + out.putVarInt32(8) + out.putVarInt32(self.width_) + out.putVarInt32(16) + out.putVarInt32(self.height_) + out.putVarInt32(26) + out.putVarInt32(self.output_.ByteSize()) + self.output_.OutputUnchecked(out) + if (self.has_color_): + out.putVarInt32(32) + out.putVarInt32(self.color_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 8: + self.set_width(d.getVarInt32()) + continue + if tt == 16: + self.set_height(d.getVarInt32()) + continue + if tt == 26: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_output().TryMerge(tmp) + continue + if tt == 32: + self.set_color(d.getVarInt32()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_width_: res+=prefix+("width: %s\n" % self.DebugFormatInt32(self.width_)) + if self.has_height_: res+=prefix+("height: %s\n" % self.DebugFormatInt32(self.height_)) + if self.has_output_: + res+=prefix+"output <\n" + res+=self.output_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + if self.has_color_: res+=prefix+("color: %s\n" % self.DebugFormatInt32(self.color_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kwidth = 1 + kheight = 2 + koutput = 3 + kcolor = 4 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "width", + 2: "height", + 3: "output", + 4: "color", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ImagesCompositeRequest(ProtocolBuffer.ProtocolMessage): + has_canvas_ = 0 + + def __init__(self, contents=None): + self.image_ = [] + self.options_ = [] + self.canvas_ = ImagesCanvas() + if contents is not None: self.MergeFromString(contents) + + def image_size(self): return len(self.image_) + def image_list(self): return self.image_ + + def image(self, i): + return self.image_[i] + + def mutable_image(self, i): + return self.image_[i] + + def add_image(self): + x = ImageData() + self.image_.append(x) + return x + + def clear_image(self): + self.image_ = [] + def options_size(self): return len(self.options_) + def options_list(self): return self.options_ + + def options(self, i): + return self.options_[i] + + def mutable_options(self, i): + return self.options_[i] + + def add_options(self): + x = CompositeImageOptions() + self.options_.append(x) + return x + + def clear_options(self): + self.options_ = [] + def canvas(self): return self.canvas_ + + def mutable_canvas(self): self.has_canvas_ = 1; return self.canvas_ + + def clear_canvas(self):self.has_canvas_ = 0; self.canvas_.Clear() + + def has_canvas(self): return self.has_canvas_ + + + def MergeFrom(self, x): + assert x is not self + for i in xrange(x.image_size()): self.add_image().CopyFrom(x.image(i)) + for i in xrange(x.options_size()): self.add_options().CopyFrom(x.options(i)) + if (x.has_canvas()): self.mutable_canvas().MergeFrom(x.canvas()) + + def Equals(self, x): + if x is self: return 1 + if len(self.image_) != len(x.image_): return 0 + for e1, e2 in zip(self.image_, x.image_): + if e1 != e2: return 0 + if len(self.options_) != len(x.options_): return 0 + for e1, e2 in zip(self.options_, x.options_): + if e1 != e2: return 0 + if self.has_canvas_ != x.has_canvas_: return 0 + if self.has_canvas_ and self.canvas_ != x.canvas_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + for p in self.image_: + if not p.IsInitialized(debug_strs): initialized=0 + for p in self.options_: + if not p.IsInitialized(debug_strs): initialized=0 + if (not self.has_canvas_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: canvas not set.') + elif not self.canvas_.IsInitialized(debug_strs): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += 1 * len(self.image_) + for i in xrange(len(self.image_)): n += self.lengthString(self.image_[i].ByteSize()) + n += 1 * len(self.options_) + for i in xrange(len(self.options_)): n += self.lengthString(self.options_[i].ByteSize()) + n += self.lengthString(self.canvas_.ByteSize()) + return n + 1 + + def Clear(self): + self.clear_image() + self.clear_options() + self.clear_canvas() + + def OutputUnchecked(self, out): + for i in xrange(len(self.image_)): + out.putVarInt32(10) + out.putVarInt32(self.image_[i].ByteSize()) + self.image_[i].OutputUnchecked(out) + for i in xrange(len(self.options_)): + out.putVarInt32(18) + out.putVarInt32(self.options_[i].ByteSize()) + self.options_[i].OutputUnchecked(out) + out.putVarInt32(26) + out.putVarInt32(self.canvas_.ByteSize()) + self.canvas_.OutputUnchecked(out) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.add_image().TryMerge(tmp) + continue + if tt == 18: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.add_options().TryMerge(tmp) + continue + if tt == 26: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_canvas().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + cnt=0 + for e in self.image_: + elm="" + if printElemNumber: elm="(%d)" % cnt + res+=prefix+("image%s <\n" % elm) + res+=e.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + cnt+=1 + cnt=0 + for e in self.options_: + elm="" + if printElemNumber: elm="(%d)" % cnt + res+=prefix+("options%s <\n" % elm) + res+=e.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + cnt+=1 + if self.has_canvas_: + res+=prefix+"canvas <\n" + res+=self.canvas_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kimage = 1 + koptions = 2 + kcanvas = 3 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "image", + 2: "options", + 3: "canvas", + }, 3) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + }, 3, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ImagesCompositeResponse(ProtocolBuffer.ProtocolMessage): has_image_ = 0 def __init__(self, contents=None): @@ -914,20 +1659,362 @@ class ImagesTransformResponse(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kimage = 1 - _TEXT = ( - "ErrorCode", - "image", - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "image", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ImagesHistogramRequest(ProtocolBuffer.ProtocolMessage): + has_image_ = 0 + + def __init__(self, contents=None): + self.image_ = ImageData() + if contents is not None: self.MergeFromString(contents) + + def image(self): return self.image_ + + def mutable_image(self): self.has_image_ = 1; return self.image_ + + def clear_image(self):self.has_image_ = 0; self.image_.Clear() + + def has_image(self): return self.has_image_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_image()): self.mutable_image().MergeFrom(x.image()) + + def Equals(self, x): + if x is self: return 1 + if self.has_image_ != x.has_image_: return 0 + if self.has_image_ and self.image_ != x.image_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_image_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: image not set.') + elif not self.image_.IsInitialized(debug_strs): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(self.image_.ByteSize()) + return n + 1 + + def Clear(self): + self.clear_image() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putVarInt32(self.image_.ByteSize()) + self.image_.OutputUnchecked(out) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_image().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_image_: + res+=prefix+"image <\n" + res+=self.image_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kimage = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "image", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ImagesHistogram(ProtocolBuffer.ProtocolMessage): + + def __init__(self, contents=None): + self.red_ = [] + self.green_ = [] + self.blue_ = [] + if contents is not None: self.MergeFromString(contents) + + def red_size(self): return len(self.red_) + def red_list(self): return self.red_ + + def red(self, i): + return self.red_[i] + + def set_red(self, i, x): + self.red_[i] = x + + def add_red(self, x): + self.red_.append(x) + + def clear_red(self): + self.red_ = [] + + def green_size(self): return len(self.green_) + def green_list(self): return self.green_ + + def green(self, i): + return self.green_[i] + + def set_green(self, i, x): + self.green_[i] = x + + def add_green(self, x): + self.green_.append(x) + + def clear_green(self): + self.green_ = [] + + def blue_size(self): return len(self.blue_) + def blue_list(self): return self.blue_ + + def blue(self, i): + return self.blue_[i] + + def set_blue(self, i, x): + self.blue_[i] = x + + def add_blue(self, x): + self.blue_.append(x) + + def clear_blue(self): + self.blue_ = [] + + + def MergeFrom(self, x): + assert x is not self + for i in xrange(x.red_size()): self.add_red(x.red(i)) + for i in xrange(x.green_size()): self.add_green(x.green(i)) + for i in xrange(x.blue_size()): self.add_blue(x.blue(i)) + + def Equals(self, x): + if x is self: return 1 + if len(self.red_) != len(x.red_): return 0 + for e1, e2 in zip(self.red_, x.red_): + if e1 != e2: return 0 + if len(self.green_) != len(x.green_): return 0 + for e1, e2 in zip(self.green_, x.green_): + if e1 != e2: return 0 + if len(self.blue_) != len(x.blue_): return 0 + for e1, e2 in zip(self.blue_, x.blue_): + if e1 != e2: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + return initialized + + def ByteSize(self): + n = 0 + n += 1 * len(self.red_) + for i in xrange(len(self.red_)): n += self.lengthVarInt64(self.red_[i]) + n += 1 * len(self.green_) + for i in xrange(len(self.green_)): n += self.lengthVarInt64(self.green_[i]) + n += 1 * len(self.blue_) + for i in xrange(len(self.blue_)): n += self.lengthVarInt64(self.blue_[i]) + return n + 0 + + def Clear(self): + self.clear_red() + self.clear_green() + self.clear_blue() + + def OutputUnchecked(self, out): + for i in xrange(len(self.red_)): + out.putVarInt32(8) + out.putVarInt32(self.red_[i]) + for i in xrange(len(self.green_)): + out.putVarInt32(16) + out.putVarInt32(self.green_[i]) + for i in xrange(len(self.blue_)): + out.putVarInt32(24) + out.putVarInt32(self.blue_[i]) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 8: + self.add_red(d.getVarInt32()) + continue + if tt == 16: + self.add_green(d.getVarInt32()) + continue + if tt == 24: + self.add_blue(d.getVarInt32()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + cnt=0 + for e in self.red_: + elm="" + if printElemNumber: elm="(%d)" % cnt + res+=prefix+("red%s: %s\n" % (elm, self.DebugFormatInt32(e))) + cnt+=1 + cnt=0 + for e in self.green_: + elm="" + if printElemNumber: elm="(%d)" % cnt + res+=prefix+("green%s: %s\n" % (elm, self.DebugFormatInt32(e))) + cnt+=1 + cnt=0 + for e in self.blue_: + elm="" + if printElemNumber: elm="(%d)" % cnt + res+=prefix+("blue%s: %s\n" % (elm, self.DebugFormatInt32(e))) + cnt+=1 + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kred = 1 + kgreen = 2 + kblue = 3 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "red", + 2: "green", + 3: "blue", + }, 3) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.NUMERIC, + }, 3, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ImagesHistogramResponse(ProtocolBuffer.ProtocolMessage): + has_histogram_ = 0 + + def __init__(self, contents=None): + self.histogram_ = ImagesHistogram() + if contents is not None: self.MergeFromString(contents) + + def histogram(self): return self.histogram_ + + def mutable_histogram(self): self.has_histogram_ = 1; return self.histogram_ + + def clear_histogram(self):self.has_histogram_ = 0; self.histogram_.Clear() + + def has_histogram(self): return self.has_histogram_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_histogram()): self.mutable_histogram().MergeFrom(x.histogram()) + + def Equals(self, x): + if x is self: return 1 + if self.has_histogram_ != x.has_histogram_: return 0 + if self.has_histogram_ and self.histogram_ != x.histogram_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_histogram_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: histogram not set.') + elif not self.histogram_.IsInitialized(debug_strs): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(self.histogram_.ByteSize()) + return n + 1 + + def Clear(self): + self.clear_histogram() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putVarInt32(self.histogram_.ByteSize()) + self.histogram_.OutputUnchecked(out) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_histogram().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_histogram_: + res+=prefix+"histogram <\n" + res+=self.histogram_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + khistogram = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "histogram", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" -__all__ = ['ImagesServiceError','ImagesServiceTransform','Transform','ImageData','OutputSettings','ImagesTransformRequest','ImagesTransformResponse'] +__all__ = ['ImagesServiceError','ImagesServiceTransform','Transform','ImageData','OutputSettings','ImagesTransformRequest','ImagesTransformResponse','CompositeImageOptions','ImagesCanvas','ImagesCompositeRequest','ImagesCompositeResponse','ImagesHistogramRequest','ImagesHistogram','ImagesHistogramResponse'] google_appengine/google/appengine/api/images/images_service_pb.py @@ -22,9 +22,13 @@ import logging import StringIO -import PIL -from PIL import _imaging -from PIL import Image +try: + import PIL + from PIL import _imaging + from PIL import Image +except ImportError: + import _imaging + import Image from google.appengine.api import apiproxy_stub from google.appengine.api import images @@ -32,6 +36,22 @@ from google.appengine.api.images import images_service_pb from google.appengine.runtime import apiproxy_errors +def _ArgbToRgbaTuple(argb): + """Convert from a single ARGB value to a tuple containing RGBA. + + Args: + argb: Signed 32 bit integer containing an ARGB value. + + Returns: + RGBA tuple. + """ + unsigned_argb = argb % 0x100000000 + return ((unsigned_argb >> 16) & 0xFF, + (unsigned_argb >> 8) & 0xFF, + unsigned_argb & 0xFF, + (unsigned_argb >> 24) & 0xFF) + + class ImagesServiceStub(apiproxy_stub.APIProxyStub): """Stub version of images API to be used with the dev_appserver.""" @@ -44,32 +64,101 @@ class ImagesServiceStub(apiproxy_stub.APIProxyStub): super(ImagesServiceStub, self).__init__(service_name) Image.init() - def _Dynamic_Transform(self, request, response): - """Trivial implementation of ImagesService::Transform. + def _Dynamic_Composite(self, request, response): + """Implementation of ImagesService::Composite. Based off documentation of the PIL library at http://www.pythonware.com/library/pil/handbook/index.htm Args: - request: ImagesTransformRequest, contains image request info. - response: ImagesTransformResponse, contains transformed image. + request: ImagesCompositeRequest, contains image request info. + response: ImagesCompositeResponse, contains transformed image. """ - image = request.image().content() - if not image: + width = request.canvas().width() + height = request.canvas().height() + color = _ArgbToRgbaTuple(request.canvas().color()) + canvas = Image.new("RGBA", (width, height), color) + sources = [] + if (not request.canvas().width() or request.canvas().width() > 4000 or + not request.canvas().height() or request.canvas().height() > 4000): raise apiproxy_errors.ApplicationError( - images_service_pb.ImagesServiceError.NOT_IMAGE) - - image = StringIO.StringIO(image) - try: - original_image = Image.open(image) - except IOError: + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) + if not request.image_size(): raise apiproxy_errors.ApplicationError( - images_service_pb.ImagesServiceError.BAD_IMAGE_DATA) + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) + if not request.options_size(): + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) + if request.options_size() > images.MAX_COMPOSITES_PER_REQUEST: + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) + for image in request.image_list(): + sources.append(self._OpenImage(image.content())) - img_format = original_image.format + for options in request.options_list(): + if (options.anchor() < images.TOP_LEFT or + options.anchor() > images.BOTTOM_RIGHT): + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) + if options.source_index() >= len(sources) or options.source_index() < 0: + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) + if options.opacity() < 0 or options.opacity() > 1: + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) + source = sources[options.source_index()] + x_anchor = (options.anchor() % 3) * 0.5 + y_anchor = (options.anchor() / 3) * 0.5 + x_offset = int(options.x_offset() + x_anchor * (width - source.size[0])) + y_offset = int(options.y_offset() + y_anchor * (height - source.size[1])) + alpha = options.opacity() * 255 + mask = Image.new("L", source.size, alpha) + canvas.paste(source, (x_offset, y_offset), mask) + response_value = self._EncodeImage(canvas, request.canvas().output()) + response.mutable_image().set_content(response_value) + + def _Dynamic_Histogram(self, request, response): + """Trivial implementation of ImagesService::Histogram. + + Based off documentation of the PIL library at + http://www.pythonware.com/library/pil/handbook/index.htm + + Args: + request: ImagesHistogramRequest, contains the image. + response: ImagesHistogramResponse, contains histogram of the image. + """ + image = self._OpenImage(request.image().content()) + img_format = image.format if img_format not in ("BMP", "GIF", "ICO", "JPEG", "PNG", "TIFF"): raise apiproxy_errors.ApplicationError( images_service_pb.ImagesServiceError.NOT_IMAGE) + image = image.convert("RGBA") + red = [0] * 256 + green = [0] * 256 + blue = [0] * 256 + for pixel in image.getdata(): + red[int((pixel[0] * pixel[3]) / 255)] += 1 + green[int((pixel[1] * pixel[3]) / 255)] += 1 + blue[int((pixel[2] * pixel[3]) / 255)] += 1 + histogram = response.mutable_histogram() + for value in red: + histogram.add_red(value) + for value in green: + histogram.add_green(value) + for value in blue: + histogram.add_blue(value) + + def _Dynamic_Transform(self, request, response): + """Trivial implementation of ImagesService::Transform. + + Based off documentation of the PIL library at + http://www.pythonware.com/library/pil/handbook/index.htm + + Args: + request: ImagesTransformRequest, contains image request info. + response: ImagesTransformResponse, contains transformed image. + """ + original_image = self._OpenImage(request.image().content()) new_image = self._ProcessTransforms(original_image, request.transform_list()) @@ -100,6 +189,36 @@ class ImagesServiceStub(apiproxy_stub.APIProxyStub): return image_string.getvalue() + def _OpenImage(self, image): + """Opens an image provided as a string. + + Args: + image: image data to be opened + + Raises: + apiproxy_errors.ApplicationError if the image cannot be opened or if it + is an unsupported format. + + Returns: + Image containing the image data passed in. + """ + if not image: + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.NOT_IMAGE) + + image = StringIO.StringIO(image) + try: + image = Image.open(image) + except IOError: + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_IMAGE_DATA) + + img_format = image.format + if img_format not in ("BMP", "GIF", "ICO", "JPEG", "PNG", "TIFF"): + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.NOT_IMAGE) + return image + def _ValidateCropArg(self, arg): """Check an argument for the Crop transform. @@ -246,24 +365,6 @@ class ImagesServiceStub(apiproxy_stub.APIProxyStub): return image.crop(box) - def _CheckTransformCount(self, transform_map, req_transform): - """Check that the requested transform hasn't already been set in map. - - Args: - transform_map: {images_service_pb.ImagesServiceTransform: boolean}, map - to use to determine if the requested transform has been called. - req_transform: images_service_pb.ImagesServiceTransform, the requested - transform. - - Raises: - BadRequestError if we are passed more than one of the same type of - transform. - """ - if req_transform in transform_map: - raise apiproxy_errors.ApplicationError( - images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) - transform_map[req_transform] = True - def _ProcessTransforms(self, image, transforms): """Execute PIL operations based on transform values. @@ -279,56 +380,29 @@ class ImagesServiceStub(apiproxy_stub.APIProxyStub): transform. """ new_image = image - transform_map = {} + if len(transforms) > images.MAX_TRANSFORMS_PER_REQUEST: + raise apiproxy_errors.ApplicationError( + images_service_pb.ImagesServiceError.BAD_TRANSFORM_DATA) for transform in transforms: if transform.has_width() or transform.has_height(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.RESIZE - ) - new_image = self._Resize(new_image, transform) elif transform.has_rotate(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.ROTATE - ) - new_image = self._Rotate(new_image, transform) elif transform.has_horizontal_flip(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.HORIZONTAL_FLIP - ) - new_image = new_image.transpose(Image.FLIP_LEFT_RIGHT) elif transform.has_vertical_flip(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.VERTICAL_FLIP - ) - new_image = new_image.transpose(Image.FLIP_TOP_BOTTOM) elif (transform.has_crop_left_x() or transform.has_crop_top_y() or transform.has_crop_right_x() or transform.has_crop_bottom_y()): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.CROP - ) - new_image = self._Crop(new_image, transform) elif transform.has_autolevels(): - self._CheckTransformCount( - transform_map, - images_service_pb.ImagesServiceTransform.IM_FEELING_LUCKY - ) logging.info("I'm Feeling Lucky autolevels will be visible once this " "application is deployed.") else: google_appengine/google/appengine/api/images/images_stub.py @@ -25,11 +25,12 @@ for their applications. Also provides a few utility methods. + +import email from email import MIMEBase from email import MIMEMultipart from email import MIMEText -import mimetypes -import types +import logging from google.appengine.api import api_base_pb from google.appengine.api import apiproxy_stub_map @@ -39,39 +40,53 @@ from google.appengine.api.mail_errors import * from google.appengine.runtime import apiproxy_errors + ERROR_MAP = { - mail_service_pb.MailServiceError.BAD_REQUEST: - BadRequestError, + mail_service_pb.MailServiceError.BAD_REQUEST: + BadRequestError, - mail_service_pb.MailServiceError.UNAUTHORIZED_SENDER: - InvalidSenderError, + mail_service_pb.MailServiceError.UNAUTHORIZED_SENDER: + InvalidSenderError, - mail_service_pb.MailServiceError.INVALID_ATTACHMENT_TYPE: - InvalidAttachmentTypeError, + mail_service_pb.MailServiceError.INVALID_ATTACHMENT_TYPE: + InvalidAttachmentTypeError, } -EXTENSION_WHITELIST = set([ - 'bmp', - 'css', - 'csv', - 'gif', - 'html', 'htm', - 'jpeg', 'jpg', 'jpe', - 'pdf', - 'png', - 'rss', - 'text', 'txt', 'asc', 'diff', 'pot', - 'tiff', 'tif', - 'wbmp', -]) +EXTENSION_MIME_MAP = { + 'asc': 'text/plain', + 'bmp': 'image/x-ms-bmp', + 'css': 'text/css', + 'csv': 'text/csv', + 'diff': 'text/plain', + 'gif': 'image/gif', + 'htm': 'text/html', + 'html': 'text/html', + 'ics': 'text/calendar', + 'jpe': 'image/jpeg', + 'jpeg': 'image/jpeg', + 'jpg': 'image/jpeg', + 'pdf': 'application/pdf', + 'png': 'image/png', + 'pot': 'text/plain', + 'rss': 'text/rss+xml', + 'text': 'text/plain', + 'tif': 'image/tiff', + 'tiff': 'image/tiff', + 'txt': 'text/plain', + 'vcf': 'text/directory', + 'wbmp': 'image/vnd.wap.wbmp', + } + +EXTENSION_WHITELIST = frozenset(EXTENSION_MIME_MAP.iterkeys()) def invalid_email_reason(email_address, field): - """Determine reason why email is invalid + """Determine reason why email is invalid. Args: email_address: Email to check. + field: Field that is invalid. Returns: String indicating invalid email reason if there is one, @@ -82,7 +97,7 @@ def invalid_email_reason(email_address, field): if isinstance(email_address, users.User): email_address = email_address.email() - if not isinstance(email_address, types.StringTypes): + if not isinstance(email_address, basestring): return 'Invalid email address type for %s.' % field stripped_address = email_address.strip() if not stripped_address: @@ -107,10 +122,11 @@ IsEmailValid = is_email_valid def check_email_valid(email_address, field): - """Check that email is valid + """Check that email is valid. Args: email_address: Email to check. + field: Field to check. Raises: InvalidEmailError if email_address is invalid. @@ -154,7 +170,7 @@ def _email_sequence(emails): Single tuple with email in it if only one email string provided, else returns emails as is. """ - if isinstance(emails, types.StringTypes): + if isinstance(emails, basestring): return emails, return emails @@ -172,11 +188,29 @@ def _attachment_sequence(attachments): Single tuple with attachment tuple in it if only one attachment provided, else returns attachments as is. """ - if len(attachments) == 2 and isinstance(attachments[0], types.StringTypes): + if len(attachments) == 2 and isinstance(attachments[0], basestring): return attachments, return attachments +def _parse_mime_message(mime_message): + """Helper function converts a mime_message in to email.Message.Message. + + Args: + mime_message: MIME Message, string or file containing mime message. + + Returns: + Instance of email.Message.Message. Will return mime_message if already + an instance. + """ + if isinstance(mime_message, email.Message.Message): + return mime_message + elif isinstance(mime_message, basestring): + return email.message_from_string(mime_message) + else: + return email.message_from_file(mime_message) + + def send_mail(sender, to, subject, @@ -234,6 +268,35 @@ def send_mail_to_admins(sender, SendMailToAdmins = send_mail_to_admins +def _GetMimeType(file_name): + """Determine mime-type from file name. + + Parses file name and determines mime-type based on extension map. + + This method is not part of the public API and should not be used by + applications. + + Args: + file_name: File to determine extension for. + + Returns: + Mime-type associated with file extension. + + Raises: + InvalidAttachmentTypeError when the file name of an attachment. + """ + extension_index = file_name.rfind('.') + if extension_index == -1: + raise InvalidAttachmentTypeError( + "File '%s' does not have an extension" % file_name) + extension = file_name[extension_index + 1:] + mime_type = EXTENSION_MIME_MAP.get(extension, None) + if mime_type is None: + raise InvalidAttachmentTypeError( + "Extension '%s' is not supported." % extension) + return mime_type + + def mail_message_to_mime_message(protocol_message): """Generate a MIMEMultitype message from protocol buffer. @@ -245,10 +308,13 @@ def mail_message_to_mime_message(protocol_message): to a list of comma separated email addresses. Args: - message: Message PB to convert to MIMEMultitype. + protocol_message: Message PB to convert to MIMEMultitype. Returns: MIMEMultitype representing the provided MailMessage. + + Raises: + InvalidAttachmentTypeError when the file name of an attachment """ parts = [] if protocol_message.has_textbody(): @@ -264,14 +330,13 @@ def mail_message_to_mime_message(protocol_message): result = MIMEMultipart.MIMEMultipart(_subparts=payload) for attachment in protocol_message.attachment_list(): - mime_type, encoding = mimetypes.guess_type(attachment.filename()) - assert mime_type is not None + file_name = attachment.filename() + mime_type = _GetMimeType(file_name) maintype, subtype = mime_type.split('/') mime_attachment = MIMEBase.MIMEBase(maintype, subtype) mime_attachment.add_header('Content-Disposition', 'attachment', filename=attachment.filename()) - mime_attachment.set_charset(encoding) mime_attachment.set_payload(attachment.data()) result.attach(mime_attachment) @@ -283,7 +348,7 @@ def mail_message_to_mime_message(protocol_message): result['Bcc'] = ', '.join(protocol_message.bcc_list()) result['From'] = protocol_message.sender() - result['ReplyTo'] = protocol_message.replyto() + result['Reply-To'] = protocol_message.replyto() result['Subject'] = protocol_message.subject() return result @@ -292,7 +357,7 @@ MailMessageToMIMEMessage = mail_message_to_mime_message def _to_str(value): - """Helper function to make sure unicode values converted to utf-8 + """Helper function to make sure unicode values converted to utf-8. Args: value: str or unicode to convert to utf-8. @@ -304,6 +369,129 @@ def _to_str(value): return value.encode('utf-8') return value + +class EncodedPayload(object): + """Wrapper for a payload that contains encoding information. + + When an email is recieved, it is usually encoded using a certain + character set, and then possibly further encoded using a transfer + encoding in that character set. Most of the times, it is possible + to decode the encoded payload as is, however, in the case where it + is not, the encoded payload and the original encoding information + must be preserved. + + Attributes: + payload: The original encoded payload. + charset: The character set of the encoded payload. None means use + default character set. + encoding: The transfer encoding of the encoded payload. None means + content not encoded. + """ + + def __init__(self, payload, charset=None, encoding=None): + """Constructor. + + Args: + payload: Maps to attribute of the same name. + charset: Maps to attribute of the same name. + encoding: Maps to attribute of the same name. + """ + self.payload = payload + self.charset = charset + self.encoding = encoding + + def decode(self): + """Attempt to decode the encoded data. + + Attempt to use pythons codec library to decode the payload. All + exceptions are passed back to the caller. + + Returns: + Binary or unicode version of payload content. + """ + payload = self.payload + + if self.encoding and self.encoding.lower() != '7bit': + try: + payload = payload.decode(self.encoding).lower() + except LookupError: + raise UnknownEncodingError('Unknown decoding %s.' % self.encoding) + except (Exception, Error), e: + raise PayloadEncodingError('Could not decode payload: %s' % e) + + if self.charset and str(self.charset).lower() != '7bit': + try: + payload = payload.decode(str(self.charset)).lower() + except LookupError: + raise UnknownCharsetError('Unknown charset %s.' % self.charset) + except (Exception, Error), e: + raise PayloadEncodingError('Could read characters: %s' % e) + + return payload + + def __eq__(self, other): + """Equality operator. + + Args: + other: The other EncodedPayload object to compare with. Comparison + with other object types are not implemented. + + Returns: + True of payload and encodings are equal, else false. + """ + if isinstance(other, EncodedPayload): + return (self.payload == other.payload and + self.charset == other.charset and + self.encoding == other.encoding) + else: + return NotImplemented + + def copy_to(self, mime_message): + """Copy contents to MIME message payload. + + If no content transfer encoding is specified, and the character set does + not equal the over-all message encoding, the payload will be base64 + encoded. + + Args: + mime_message: Message instance to receive new payload. + """ + if self.encoding: + mime_message['content-transfer-encoding'] = self.encoding + mime_message.set_payload(self.payload, self.charset) + + def to_mime_message(self): + """Convert to MIME message. + + Returns: + MIME message instance of payload. + """ + mime_message = email.Message.Message() + self.copy_to(mime_message) + return mime_message + + def __str__(self): + """String representation of encoded message. + + Returns: + MIME encoded representation of encoded payload as an independent message. + """ + return str(self.to_mime_message()) + + def __repr__(self): + """Basic representation of encoded payload. + + Returns: + Payload itself is represented by its hash value. + """ + result = '<EncodedPayload payload=#%d' % hash(self.payload) + if self.charset: + result += ' charset=%s' % self.charset + if self.encoding: + result += ' encoding=%s' % self.encoding + return result + '>' + + class _EmailMessageBase(object): """Base class for email API service objects. @@ -312,25 +500,39 @@ class _EmailMessageBase(object): """ PROPERTIES = set([ - 'sender', - 'reply_to', - 'subject', - 'body', - 'html', - 'attachments', + 'sender', + 'reply_to', + 'subject', + 'body', + 'html', + 'attachments', ]) - def __init__(self, **kw): + PROPERTIES.update(('to', 'cc', 'bcc')) + + def __init__(self, mime_message=None, **kw): """Initialize Email message. Creates new MailMessage protocol buffer and initializes it with any keyword arguments. Args: + mime_message: MIME message to initialize from. If instance of + email.Message.Message will take ownership as original message. kw: List of keyword properties as defined by PROPERTIES. """ + if mime_message: + mime_message = _parse_mime_message(mime_message) + self.update_from_mime_message(mime_message) + self.__original = mime_message + self.initialize(**kw) + @property + def original(self): + """Get original MIME message from which values were set.""" + return self.__original + def initialize(self, **kw): """Keyword initialization. @@ -356,6 +558,7 @@ class _EmailMessageBase(object): - Subject must be set. - A recipient must be specified. - Must contain a body. + - All bodies and attachments must decode properly. This check does not include determining if the sender is actually authorized to send email for the application. @@ -368,23 +571,44 @@ class _EmailMessageBase(object): MissingSenderError: No sender specified. MissingSubjectError: Subject is not specified. MissingBodyError: No body specified. + PayloadEncodingError: Payload is not properly encoded. + UnknownEncodingError: Payload has unknown encoding. + UnknownCharsetError: Payload has unknown character set. """ if not hasattr(self, 'sender'): raise MissingSenderError() if not hasattr(self, 'subject'): raise MissingSubjectError() - if not hasattr(self, 'body') and not hasattr(self, 'html'): + + found_body = False + + try: + body = self.body + except AttributeError: + pass + else: + if isinstance(body, EncodedPayload): + body.decode() + found_body = True + + try: + html = self.html + except AttributeError: + pass + else: + if isinstance(html, EncodedPayload): + html.decode() + found_body = True + + if not found_body: raise MissingBodyError() + if hasattr(self, 'attachments'): - for filename, data in _attachment_sequence(self.attachments): - split_filename = filename.split('.') - if len(split_filename) < 2: - raise InvalidAttachmentTypeError() - if split_filename[-1] not in EXTENSION_WHITELIST: - raise InvalidAttachmentTypeError() - mime_type, encoding = mimetypes.guess_type(filename) - if mime_type is None: - raise InvalidAttachmentTypeError() + for file_name, data in _attachment_sequence(self.attachments): + _GetMimeType(file_name) + + if isinstance(data, EncodedPayload): + data.decode() def CheckInitialized(self): self.check_initialized() @@ -413,6 +637,10 @@ class _EmailMessageBase(object): Returns: MailMessage protocol version of mail message. + + Raises: + Passes through decoding errors that occur when using when decoding + EncodedPayload objects. """ self.check_initialized() message = mail_service_pb.MailMessage() @@ -421,13 +649,22 @@ class _EmailMessageBase(object): if hasattr(self, 'reply_to'): message.set_replyto(_to_str(self.reply_to)) message.set_subject(_to_str(self.subject)) + if hasattr(self, 'body'): - message.set_textbody(_to_str(self.body)) + body = self.body + if isinstance(body, EncodedPayload): + body = body.decode() + message.set_textbody(_to_str(body)) if hasattr(self, 'html'): - message.set_htmlbody(_to_str(self.html)) + html = self.html + if isinstance(html, EncodedPayload): + html = html.decode() + message.set_htmlbody(_to_str(html)) if hasattr(self, 'attachments'): for file_name, data in _attachment_sequence(self.attachments): + if isinstance(data, EncodedPayload): + data = data.decode() attachment = message.add_attachment() attachment.set_filename(_to_str(file_name)) attachment.set_data(_to_str(data)) @@ -450,7 +687,7 @@ class _EmailMessageBase(object): MissingSenderError: No sender specified. MissingSubjectError: Subject is not specified. MissingBodyError: No body specified. - """ + """ return mail_message_to_mime_message(self.ToProto()) def ToMIMEMessage(self): @@ -482,8 +719,8 @@ class _EmailMessageBase(object): def _check_attachment(self, attachment): file_name, data = attachment - if not (isinstance(file_name, types.StringTypes) or - isinstance(data, types.StringTypes)): + if not (isinstance(file_name, basestring) or + isinstance(data, basestring)): raise TypeError() def _check_attachments(self, attachments): @@ -499,7 +736,7 @@ class _EmailMessageBase(object): Raises: TypeError if values are not string type. """ - if len(attachments) == 2 and isinstance(attachments[0], types.StringTypes): + if len(attachments) == 2 and isinstance(attachments[0], basestring): self._check_attachment(attachments) else: for attachment in attachments: @@ -513,21 +750,134 @@ class _EmailMessageBase(object): Args: attr: Attribute to access. value: New value for field. + + Raises: + ValueError: If provided with an empty field. + AttributeError: If not an allowed assignment field. """ - if attr in ['sender', 'reply_to']: - check_email_valid(value, attr) + if not attr.startswith('_EmailMessageBase'): + if attr in ['sender', 'reply_to']: + check_email_valid(value, attr) - if not value: - raise ValueError('May not set empty value for \'%s\'' % attr) + if not value: + raise ValueError('May not set empty value for \'%s\'' % attr) - if attr not in self.PROPERTIES: - raise AttributeError('\'EmailMessage\' has no attribute \'%s\'' % attr) + if attr not in self.PROPERTIES: + raise AttributeError('\'EmailMessage\' has no attribute \'%s\'' % attr) - if attr == 'attachments': - self._check_attachments(value) + if attr == 'attachments': + self._check_attachments(value) super(_EmailMessageBase, self).__setattr__(attr, value) + def _add_body(self, content_type, payload): + """Add body to email from payload. + + Will overwrite any existing default plain or html body. + + Args: + content_type: Content-type of body. + payload: Payload to store body as. + """ + if content_type == 'text/plain': + self.body = payload + elif content_type == 'text/html': + self.html = payload + + def _update_payload(self, mime_message): + """Update payload of mail message from mime_message. + + This function works recusively when it receives a multipart body. + If it receives a non-multi mime object, it will determine whether or + not it is an attachment by whether it has a filename or not. Attachments + and bodies are then wrapped in EncodedPayload with the correct charsets and + encodings. + + Args: + mime_message: A Message MIME email object. + """ + payload = mime_message.get_payload() + + if payload: + if mime_message.get_content_maintype() == 'multipart': + for alternative in payload: + self._update_payload(alternative) + else: + filename = mime_message.get_param('filename', + header='content-disposition') + if not filename: + filename = mime_message.get_param('name') + + payload = EncodedPayload(payload, + mime_message.get_charset(), + mime_message['content-transfer-encoding']) + + if filename: + try: + attachments = self.attachments + except AttributeError: + self.attachments = (filename, payload) + else: + if isinstance(attachments[0], basestring): + self.attachments = [attachments] + attachments = self.attachments + attachments.append((filename, payload)) + else: + self._add_body(mime_message.get_content_type(), payload) + + def update_from_mime_message(self, mime_message): + """Copy information from a mime message. + + Set information of instance to values of mime message. This method + will only copy values that it finds. Any missing values will not + be copied, nor will they overwrite old values with blank values. + + This object is not guaranteed to be initialized after this call. + + Args: + mime_message: email.Message instance to copy information from. + + Returns: + MIME Message instance of mime_message argument. + """ + mime_message = _parse_mime_message(mime_message) + + sender = mime_message['from'] + if sender: + self.sender = sender + + reply_to = mime_message['reply-to'] + if reply_to: + self.reply_to = reply_to + + subject = mime_message['subject'] + if subject: + self.subject = subject + + self._update_payload(mime_message) + + def bodies(self, content_type=None): + """Iterate over all bodies. + + Yields: + Tuple (content_type, payload) for html and body in that order. + """ + if (not content_type or + content_type == 'text' or + content_type == 'text/html'): + try: + yield 'text/html', self.html + except AttributeError: + pass + + if (not content_type or + content_type == 'text' or + content_type == 'text/plain'): + try: + yield 'text/plain', self.body + except AttributeError: + pass + class EmailMessage(_EmailMessageBase): """Main interface to email API service. @@ -557,8 +907,7 @@ class EmailMessage(_EmailMessageBase): """ _API_CALL = 'Send' - PROPERTIES = _EmailMessageBase.PROPERTIES - PROPERTIES.update(('to', 'cc', 'bcc')) + PROPERTIES = set(_EmailMessageBase.PROPERTIES) def check_initialized(self): """Provide additional checks to ensure recipients have been specified. @@ -594,13 +943,46 @@ class EmailMessage(_EmailMessageBase): def __setattr__(self, attr, value): """Provides additional checks on recipient fields.""" if attr in ['to', 'cc', 'bcc']: - if isinstance(value, types.StringTypes): + if isinstance(value, basestring): check_email_valid(value, attr) else: - _email_check_and_list(value, attr) + for address in value: + check_email_valid(address, attr) super(EmailMessage, self).__setattr__(attr, value) + def update_from_mime_message(self, mime_message): + """Copy information from a mime message. + + Update fields for recipients. + + Args: + mime_message: email.Message instance to copy information from. + """ + mime_message = _parse_mime_message(mime_message) + super(EmailMessage, self).update_from_mime_message(mime_message) + + to = mime_message.get_all('to') + if to: + if len(to) == 1: + self.to = to[0] + else: + self.to = to + + cc = mime_message.get_all('cc') + if cc: + if len(cc) == 1: + self.cc = cc[0] + else: + self.cc = cc + + bcc = mime_message.get_all('bcc') + if bcc: + if len(bcc) == 1: + self.bcc = bcc[0] + else: + self.bcc = bcc + class AdminEmailMessage(_EmailMessageBase): """Interface to sending email messages to all admins via the amil API. @@ -632,3 +1014,132 @@ class AdminEmailMessage(_EmailMessageBase): """ _API_CALL = 'SendToAdmins' + __UNUSED_PROPERTIES = set(('to', 'cc', 'bcc')) + + def __setattr__(self, attr, value): + if attr in self.__UNUSED_PROPERTIES: + logging.warning('\'%s\' is not a valid property to set ' + 'for AdminEmailMessage. It is unused.', attr) + super(AdminEmailMessage, self).__setattr__(attr, value) + + +class InboundEmailMessage(EmailMessage): + """Parsed email object as recevied from external source. + + Has a date field and can store any number of additional bodies. These + additional attributes make the email more flexible as required for + incoming mail, where the developer has less control over the content. + + Example Usage: + + # Read mail message from CGI input. + message = InboundEmailMessage(sys.stdin.read()) + logging.info('Received email message from %s at %s', + message.sender, + message.date) + enriched_body = list(message.bodies('text/enriched'))[0] + ... Do something with body ... + """ + + __HEADER_PROPERTIES = {'date': 'date', + 'message_id': 'message-id', + } + + PROPERTIES = frozenset(_EmailMessageBase.PROPERTIES | + set(('alternate_bodies',)) | + set(__HEADER_PROPERTIES.iterkeys())) + + def update_from_mime_message(self, mime_message): + """Update values from MIME message. + + Copies over date values. + + Args: + mime_message: email.Message instance to copy information from. + """ + mime_message = _parse_mime_message(mime_message) + super(InboundEmailMessage, self).update_from_mime_message(mime_message) + + for property, header in InboundEmailMessage.__HEADER_PROPERTIES.iteritems(): + value = mime_message[header] + if value: + setattr(self, property, value) + + def _add_body(self, content_type, payload): + """Add body to inbound message. + + Method is overidden to handle incoming messages that have more than one + plain or html bodies or has any unidentified bodies. + + This method will not overwrite existing html and body values. This means + that when updating, the text and html bodies that are first in the MIME + document order are assigned to the body and html properties. + + Args: + content_type: Content-type of additional body. + payload: Content of additional body. + """ + if (content_type == 'text/plain' and not hasattr(self, 'body') or + content_type == 'text/html' and not hasattr(self, 'html')): + super(InboundEmailMessage, self)._add_body(content_type, payload) + else: + try: + alternate_bodies = self.alternate_bodies + except AttributeError: + alternate_bodies = self.alternate_bodies = [(content_type, payload)] + else: + alternate_bodies.append((content_type, payload)) + + def bodies(self, content_type=None): + """Iterate over all bodies. + + Args: + content_type: Content type to filter on. Allows selection of only + specific types of content. Can be just the base type of the content + type. For example: + content_type = 'text/html' # Matches only HTML content. + content_type = 'text' # Matches text of any kind. + + Yields: + Tuple (content_type, payload) for all bodies of message, including body, + html and all alternate_bodies in that order. + """ + main_bodies = super(InboundEmailMessage, self).bodies(content_type) + for payload_type, payload in main_bodies: + yield payload_type, payload + + partial_type = bool(content_type and content_type.find('/') < 0) + + try: + for payload_type, payload in self.alternate_bodies: + if content_type: + if partial_type: + match_type = payload_type.split('/')[0] + else: + match_type = payload_type + match = match_type == content_type + else: + match = True + + if match: + yield payload_type, payload + except AttributeError: + pass + + def to_mime_message(self): + """Convert to MIME message. + + Adds additional headers from inbound email. + + Returns: + MIME message instance of payload. + """ + mime_message = super(InboundEmailMessage, self).to_mime_message() + + for property, header in InboundEmailMessage.__HEADER_PROPERTIES.iteritems(): + try: + mime_message[header] = getattr(self, property) + except AttributeError: + pass + + return mime_message google_appengine/google/appengine/api/mail.py @@ -44,3 +44,12 @@ class MissingSubjectError(Error): class MissingBodyError(Error): """No body specified in message.""" + +class PayloadEncodingError(Error): + """Unknown payload encoding.""" + +class UnknownEncodingError(PayloadEncodingError): + """Raised when encoding is not known.""" + +class UnknownCharsetError(PayloadEncodingError): + """Raised when charset is not known.""" google_appengine/google/appengine/api/mail_errors.py @@ -22,7 +22,7 @@ import dummy_thread as thread __pychecker__ = """maxreturns=0 maxbranches=0 no-callinit unusednames=printElemNumber,debug_strs no-special""" -from google.appengine.api.api_base_pb import VoidProto +from google.appengine.api.api_base_pb import * class MailServiceError(ProtocolBuffer.ProtocolMessage): OK = 0 @@ -81,13 +81,17 @@ class MailServiceError(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -107,8 +111,9 @@ class MailAttachment(ProtocolBuffer.ProtocolMessage): self.filename_ = x def clear_filename(self): - self.has_filename_ = 0 - self.filename_ = "" + if self.has_filename_: + self.has_filename_ = 0 + self.filename_ = "" def has_filename(self): return self.has_filename_ @@ -119,8 +124,9 @@ class MailAttachment(ProtocolBuffer.ProtocolMessage): self.data_ = x def clear_data(self): - self.has_data_ = 0 - self.data_ = "" + if self.has_data_: + self.has_data_ = 0 + self.data_ = "" def has_data(self): return self.has_data_ @@ -185,22 +191,24 @@ class MailAttachment(ProtocolBuffer.ProtocolMessage): if self.has_data_: res+=prefix+("Data: %s\n" % self.DebugFormatString(self.data_)) return res - kFileName = 1 - kData = 2 - _TEXT = ( - "ErrorCode", - "FileName", - "Data", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + kFileName = 1 + kData = 2 - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "FileName", + 2: "Data", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -230,8 +238,9 @@ class MailMessage(ProtocolBuffer.ProtocolMessage): self.sender_ = x def clear_sender(self): - self.has_sender_ = 0 - self.sender_ = "" + if self.has_sender_: + self.has_sender_ = 0 + self.sender_ = "" def has_sender(self): return self.has_sender_ @@ -242,8 +251,9 @@ class MailMessage(ProtocolBuffer.ProtocolMessage): self.replyto_ = x def clear_replyto(self): - self.has_replyto_ = 0 - self.replyto_ = "" + if self.has_replyto_: + self.has_replyto_ = 0 + self.replyto_ = "" def has_replyto(self): return self.has_replyto_ @@ -299,8 +309,9 @@ class MailMessage(ProtocolBuffer.ProtocolMessage): self.subject_ = x def clear_subject(self): - self.has_subject_ = 0 - self.subject_ = "" + if self.has_subject_: + self.has_subject_ = 0 + self.subject_ = "" def has_subject(self): return self.has_subject_ @@ -311,8 +322,9 @@ class MailMessage(ProtocolBuffer.ProtocolMessage): self.textbody_ = x def clear_textbody(self): - self.has_textbody_ = 0 - self.textbody_ = "" + if self.has_textbody_: + self.has_textbody_ = 0 + self.textbody_ = "" def has_textbody(self): return self.has_textbody_ @@ -323,8 +335,9 @@ class MailMessage(ProtocolBuffer.ProtocolMessage): self.htmlbody_ = x def clear_htmlbody(self): - self.has_htmlbody_ = 0 - self.htmlbody_ = "" + if self.has_htmlbody_: + self.has_htmlbody_ = 0 + self.htmlbody_ = "" def has_htmlbody(self): return self.has_htmlbody_ @@ -525,6 +538,10 @@ class MailMessage(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kSender = 1 kReplyTo = 2 kTo = 3 @@ -535,40 +552,31 @@ class MailMessage(ProtocolBuffer.ProtocolMessage): kHtmlBody = 8 kAttachment = 9 - _TEXT = ( - "ErrorCode", - "Sender", - "ReplyTo", - "To", - "Cc", - "Bcc", - "Subject", - "TextBody", - "HtmlBody", - "Attachment", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Sender", + 2: "ReplyTo", + 3: "To", + 4: "Cc", + 5: "Bcc", + 6: "Subject", + 7: "TextBody", + 8: "HtmlBody", + 9: "Attachment", + }, 9) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.STRING, + 5: ProtocolBuffer.Encoder.STRING, + 6: ProtocolBuffer.Encoder.STRING, + 7: ProtocolBuffer.Encoder.STRING, + 8: ProtocolBuffer.Encoder.STRING, + 9: ProtocolBuffer.Encoder.STRING, + }, 9, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" google_appengine/google/appengine/api/mail_service_pb.py @@ -196,6 +196,8 @@ class MailServiceStub(apiproxy_stub.APIProxyStub): if self._smtp_host and self._enable_sendmail: log('Both SMTP and sendmail are enabled. Ignoring sendmail.') + import email + mime_message = mail.MailMessageToMIMEMessage(request) if self._smtp_host: self._SendSMTP(mime_message, smtp_lib) google_appengine/google/appengine/api/mail_stub.py @@ -28,9 +28,12 @@ import cStringIO import math import pickle import types +import sha from google.appengine.api import api_base_pb from google.appengine.api import apiproxy_stub_map +from google.appengine.api import capabilities +from google.appengine.api import namespace_manager from google.appengine.api.memcache import memcache_service_pb from google.appengine.runtime import apiproxy_errors @@ -76,6 +79,8 @@ TYPE_INT = 3 TYPE_LONG = 4 TYPE_BOOL = 5 +CAPABILITY = capabilities.CapabilitySet('memcache') + def _key_string(key, key_prefix='', server_to_user_dict=None): """Utility function to handle different ways of requesting keys. @@ -90,14 +95,14 @@ def _key_string(key, key_prefix='', server_to_user_dict=None): (which does not have the prefix). Returns: - The key as a non-unicode string prepended with key_prefix. This is the key - sent to and stored by the server. + The key as a non-unicode string prepended with key_prefix. This is + the key sent to and stored by the server. If the resulting key is + longer then MAX_KEY_SIZE, it will be hashed with sha1 and will be + replaced with the hex representation of the said hash. Raises: TypeError: If provided key isn't a string or tuple of (int, string) or key_prefix or server_to_user_dict are of the wrong type. - ValueError: If the key, when translated to the server key, is more than - 250 bytes in length. """ if type(key) is types.TupleType: key = key[1] @@ -112,8 +117,7 @@ def _key_string(key, key_prefix='', server_to_user_dict=None): server_key = server_key.encode('utf-8') if len(server_key) > MAX_KEY_SIZE: - raise ValueError('Keys may not be more than %d bytes in length, ' - 'received %d bytes' % (MAX_KEY_SIZE, len(server_key))) + server_key = sha.new(server_key).hexdigest() if server_to_user_dict is not None: if not isinstance(server_to_user_dict, dict): @@ -345,7 +349,14 @@ class Client(object): return None if not response.has_stats(): - return None + return { + STAT_HITS: 0, + STAT_MISSES: 0, + STAT_BYTE_HITS: 0, + STAT_ITEMS: 0, + STAT_BYTES: 0, + STAT_OLDEST_ITEM_AGES: 0, + } stats = response.stats() return { @@ -371,7 +382,7 @@ class Client(object): return False return True - def get(self, key): + def get(self, key, namespace=None): """Looks up a single key in memcache. If you have multiple items to load, though, it's much more efficient @@ -382,12 +393,15 @@ class Client(object): Args: key: The key in memcache to look up. See docs on Client for details of format. + namespace: a string specifying an optional namespace to use in + the request. Returns: The value of the key, if found in memcache, else None. """ request = MemcacheGetRequest() request.add_key(_key_string(key)) + namespace_manager._add_name_space(request, namespace) response = MemcacheGetResponse() try: self._make_sync_call('memcache', 'Get', request, response) @@ -401,7 +415,7 @@ class Client(object): response.item(0).flags(), self._do_unpickle) - def get_multi(self, keys, key_prefix=''): + def get_multi(self, keys, key_prefix='', namespace=None): """Looks up multiple keys from memcache in one operation. This is the recommended way to do bulk loads. @@ -414,6 +428,8 @@ class Client(object): and not in any particular encoding. key_prefix: Prefix to prepend to all keys when talking to the server; not included in the returned dictionary. + namespace: a string specifying an optional namespace to use in + the request. Returns: A dictionary of the keys and values that were present in memcache. @@ -421,6 +437,7 @@ class Client(object): the keys in the returned dictionary. """ request = MemcacheGetRequest() + namespace_manager._add_name_space(request, namespace) response = MemcacheGetResponse() user_key = {} for key in keys: @@ -437,7 +454,7 @@ class Client(object): return_value[user_key[returned_item.key()]] = value return return_value - def delete(self, key, seconds=0): + def delete(self, key, seconds=0, namespace=None): """Deletes a key from memcache. Args: @@ -448,6 +465,8 @@ class Client(object): items can be immediately added. With or without this option, a 'set' operation will always work. Float values will be rounded up to the nearest whole second. + namespace: a string specifying an optional namespace to use in + the request. Returns: DELETE_NETWORK_FAILURE (0) on network failure, @@ -463,6 +482,7 @@ class Client(object): raise ValueError('Delete timeout must be non-negative.') request = MemcacheDeleteRequest() + namespace_manager._add_name_space(request, namespace) response = MemcacheDeleteResponse() delete_item = request.add_item() @@ -480,7 +500,7 @@ class Client(object): return DELETE_ITEM_MISSING assert False, 'Unexpected deletion status code.' - def delete_multi(self, keys, seconds=0, key_prefix=''): + def delete_multi(self, keys, seconds=0, key_prefix='', namespace=None): """Delete multiple keys at once. Args: @@ -493,6 +513,8 @@ class Client(object): the nearest whole second. key_prefix: Prefix to put on all keys when sending specified keys to memcache. See docs for get_multi() and set_multi(). + namespace: a string specifying an optional namespace to use in + the request. Returns: True if all operations completed successfully. False if one @@ -504,6 +526,7 @@ class Client(object): raise ValueError('Delete timeout must not be negative.') request = MemcacheDeleteRequest() + namespace_manager._add_name_space(request, namespace) response = MemcacheDeleteResponse() for key in keys: @@ -516,7 +539,7 @@ class Client(object): return False return True - def set(self, key, value, time=0, min_compress_len=0): + def set(self, key, value, time=0, min_compress_len=0, namespace=None): """Sets a key's value, regardless of previous contents in cache. Unlike add() and replace(), this method always sets (or @@ -532,13 +555,16 @@ class Client(object): memory pressure. Float values will be rounded up to the nearest whole second. min_compress_len: Ignored option for compatibility. + namespace: a string specifying an optional namespace to use in + the request. Returns: True if set. False on error. """ - return self._set_with_policy(MemcacheSetRequest.SET, key, value, time=time) + return self._set_with_policy(MemcacheSetRequest.SET, key, value, time=time, + namespace=namespace) - def add(self, key, value, time=0, min_compress_len=0): + def add(self, key, value, time=0, min_compress_len=0, namespace=None): """Sets a key's value, iff item is not already in memcache. Args: @@ -550,13 +576,16 @@ class Client(object): memory pressure. Float values will be rounded up to the nearest whole second. min_compress_len: Ignored option for compatibility. + namespace: a string specifying an optional namespace to use in + the request. Returns: True if added. False on error. """ - return self._set_with_policy(MemcacheSetRequest.ADD, key, value, time=time) + return self._set_with_policy(MemcacheSetRequest.ADD, key, value, time=time, + namespace=namespace) - def replace(self, key, value, time=0, min_compress_len=0): + def replace(self, key, value, time=0, min_compress_len=0, namespace=None): """Replaces a key's value, failing if item isn't already in memcache. Args: @@ -568,14 +597,16 @@ class Client(object): memory pressure. Float values will be rounded up to the nearest whole second. min_compress_len: Ignored option for compatibility. + namespace: a string specifying an optional namespace to use in + the request. Returns: True if replaced. False on RPC error or cache miss. """ return self._set_with_policy(MemcacheSetRequest.REPLACE, - key, value, time=time) + key, value, time=time, namespace=namespace) - def _set_with_policy(self, policy, key, value, time=0): + def _set_with_policy(self, policy, key, value, time=0, namespace=None): """Sets a single key with a specified policy. Helper function for set(), add(), and replace(). @@ -585,6 +616,8 @@ class Client(object): key: Key to add, set, or replace. See docs on Client for details. value: Value to set. time: Expiration time, defaulting to 0 (never expiring). + namespace: a string specifying an optional namespace to use in + the request. Returns: True if stored, False on RPC error or policy error, e.g. a replace @@ -604,6 +637,7 @@ class Client(object): item.set_flags(flags) item.set_set_policy(policy) item.set_expiration_time(int(math.ceil(time))) + namespace_manager._add_name_space(request, namespace) response = MemcacheSetResponse() try: self._make_sync_call('memcache', 'Set', request, response) @@ -613,7 +647,8 @@ class Client(object): return False return response.set_status(0) == MemcacheSetResponse.STORED - def _set_multi_with_policy(self, policy, mapping, time=0, key_prefix=''): + def _set_multi_with_policy(self, policy, mapping, time=0, key_prefix='', + namespace=None): """Set multiple keys with a specified policy. Helper function for set_multi(), add_multi(), and replace_multi(). This @@ -628,6 +663,8 @@ class Client(object): memory pressure. Float values will be rounded up to the nearest whole second. key_prefix: Prefix for to prepend to all keys. + namespace: a string specifying an optional namespace to use in + the request. Returns: A list of keys whose values were NOT set. On total success, @@ -654,6 +691,7 @@ class Client(object): item.set_flags(flags) item.set_set_policy(policy) item.set_expiration_time(int(math.ceil(time))) + namespace_manager._add_name_space(request, namespace) response = MemcacheSetResponse() try: @@ -670,7 +708,8 @@ class Client(object): return unset_list - def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0): + def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, + namespace=None): """Set multiple keys' values at once, regardless of previous contents. Args: @@ -682,15 +721,19 @@ class Client(object): whole second. key_prefix: Prefix for to prepend to all keys. min_compress_len: Unimplemented compatibility option. + namespace: a string specifying an optional namespace to use in + the request. Returns: A list of keys whose values were NOT set. On total success, this list should be empty. """ return self._set_multi_with_policy(MemcacheSetRequest.SET, mapping, - time=time, key_prefix=key_prefix) + time=time, key_prefix=key_prefix, + namespace=namespace) - def add_multi(self, mapping, time=0, key_prefix='', min_compress_len=0): + def add_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, + namespace=None): """Set multiple keys' values iff items are not already in memcache. Args: @@ -702,15 +745,19 @@ class Client(object): whole second. key_prefix: Prefix for to prepend to all keys. min_compress_len: Unimplemented compatibility option. + namespace: a string specifying an optional namespace to use in + the request. Returns: A list of keys whose values were NOT set because they did not already exist in memcache. On total success, this list should be empty. """ return self._set_multi_with_policy(MemcacheSetRequest.ADD, mapping, - time=time, key_prefix=key_prefix) + time=time, key_prefix=key_prefix, + namespace=namespace) - def replace_multi(self, mapping, time=0, key_prefix='', min_compress_len=0): + def replace_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, + namespace=None): """Replace multiple keys' values, failing if the items aren't in memcache. Args: @@ -722,23 +769,27 @@ class Client(object): whole second. key_prefix: Prefix for to prepend to all keys. min_compress_len: Unimplemented compatibility option. + namespace: a string specifying an optional namespace to use in + the request. Returns: A list of keys whose values were NOT set because they already existed in memcache. On total success, this list should be empty. """ return self._set_multi_with_policy(MemcacheSetRequest.REPLACE, mapping, - time=time, key_prefix=key_prefix) + time=time, key_prefix=key_prefix, + namespace=namespace) - def incr(self, key, delta=1): + def incr(self, key, delta=1, namespace=None, initial_value=None): """Atomically increments a key's value. Internally, the value is a unsigned 64-bit integer. Memcache doesn't check 64-bit overflows. The value, if too large, will wrap around. - The key must already exist in the cache to be incremented. To - initialize a counter, set() it to the initial value, as an + Unless an initial_value is specified, the key must already exist + in the cache to be incremented. To initialize a counter, either + specify initial_value or set() it to the initial value, as an ASCII decimal integer. Future get()s of the key, post-increment, will still be an ASCII decimal value. @@ -746,6 +797,11 @@ class Client(object): key: Key to increment. See Client's docstring for details. delta: Non-negative integer value (int or long) to increment key by, defaulting to 1. + namespace: a string specifying an optional namespace to use in + the request. + initial_value: initial value to put in the cache, if it doesn't + already exist. The default value, None, will not create a cache + entry if it doesn't already exist. Returns: New long integer value, or None if key was not in the cache, could not @@ -756,9 +812,10 @@ class Client(object): ValueError: If number is negative. TypeError: If delta isn't an int or long. """ - return self._incrdecr(key, False, delta) + return self._incrdecr(key, False, delta, namespace=namespace, + initial_value=initial_value) - def decr(self, key, delta=1): + def decr(self, key, delta=1, namespace=None, initial_value=None): """Atomically decrements a key's value. Internally, the value is a unsigned 64-bit integer. Memcache @@ -771,6 +828,11 @@ class Client(object): key: Key to decrement. See Client's docstring for details. delta: Non-negative integer value (int or long) to decrement key by, defaulting to 1. + namespace: a string specifying an optional namespace to use in + the request. + initial_value: initial value to put in the cache, if it doesn't + already exist. The default value, None, will not create a cache + entry if it doesn't already exist. Returns: New long integer value, or None if key wasn't in cache and couldn't @@ -780,9 +842,11 @@ class Client(object): ValueError: If number is negative. TypeError: If delta isn't an int or long. """ - return self._incrdecr(key, True, delta) + return self._incrdecr(key, True, delta, namespace=namespace, + initial_value=initial_value) - def _incrdecr(self, key, is_negative, delta): + def _incrdecr(self, key, is_negative, delta, namespace=None, + initial_value=None): """Increment or decrement a key by a provided delta. Args: @@ -790,6 +854,11 @@ class Client(object): is_negative: Boolean, if this is a decrement. delta: Non-negative integer amount (int or long) to increment or decrement by. + namespace: a string specifying an optional namespace to use in + the request. + initial_value: initial value to put in the cache, if it doesn't + already exist. The default value, None, will not create a cache + entry if it doesn't already exist. Returns: New long integer value, or None on cache miss or network/RPC/server @@ -805,6 +874,7 @@ class Client(object): raise ValueError('Delta must not be negative.') request = MemcacheIncrementRequest() + namespace_manager._add_name_space(request, namespace) response = MemcacheIncrementResponse() request.set_key(_key_string(key)) request.set_delta(delta) @@ -812,6 +882,8 @@ class Client(object): request.set_direction(MemcacheIncrementRequest.DECREMENT) else: request.set_direction(MemcacheIncrementRequest.INCREMENT) + if initial_value is not None: + request.set_initial_value(long(initial_value)) try: self._make_sync_call('memcache', 'Increment', request, response) google_appengine/google/appengine/api/memcache/__init__.py @@ -22,7 +22,6 @@ import dummy_thread as thread __pychecker__ = """maxreturns=0 maxbranches=0 no-callinit unusednames=printElemNumber,debug_strs no-special""" -from google.appengine.api.api_base_pb import VoidProto class MemcacheServiceError(ProtocolBuffer.ProtocolMessage): OK = 0 @@ -75,17 +74,23 @@ class MemcacheServiceError(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" class MemcacheGetRequest(ProtocolBuffer.ProtocolMessage): + has_name_space_ = 0 + name_space_ = "" def __init__(self, contents=None): self.key_ = [] @@ -106,16 +111,32 @@ class MemcacheGetRequest(ProtocolBuffer.ProtocolMessage): def clear_key(self): self.key_ = [] + def name_space(self): return self.name_space_ + + def set_name_space(self, x): + self.has_name_space_ = 1 + self.name_space_ = x + + def clear_name_space(self): + if self.has_name_space_: + self.has_name_space_ = 0 + self.name_space_ = "" + + def has_name_space(self): return self.has_name_space_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.key_size()): self.add_key(x.key(i)) + if (x.has_name_space()): self.set_name_space(x.name_space()) def Equals(self, x): if x is self: return 1 if len(self.key_) != len(x.key_): return 0 for e1, e2 in zip(self.key_, x.key_): if e1 != e2: return 0 + if self.has_name_space_ != x.has_name_space_: return 0 + if self.has_name_space_ and self.name_space_ != x.name_space_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -126,15 +147,20 @@ class MemcacheGetRequest(ProtocolBuffer.ProtocolMessage): n = 0 n += 1 * len(self.key_) for i in xrange(len(self.key_)): n += self.lengthString(len(self.key_[i])) + if (self.has_name_space_): n += 1 + self.lengthString(len(self.name_space_)) return n + 0 def Clear(self): self.clear_key() + self.clear_name_space() def OutputUnchecked(self, out): for i in xrange(len(self.key_)): out.putVarInt32(10) out.putPrefixedString(self.key_[i]) + if (self.has_name_space_): + out.putVarInt32(18) + out.putPrefixedString(self.name_space_) def TryMerge(self, d): while d.avail() > 0: @@ -142,6 +168,9 @@ class MemcacheGetRequest(ProtocolBuffer.ProtocolMessage): if tt == 10: self.add_key(d.getPrefixedString()) continue + if tt == 18: + self.set_name_space(d.getPrefixedString()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -154,20 +183,27 @@ class MemcacheGetRequest(ProtocolBuffer.ProtocolMessage): if printElemNumber: elm="(%d)" % cnt res+=prefix+("key%s: %s\n" % (elm, self.DebugFormatString(e))) cnt+=1 + if self.has_name_space_: res+=prefix+("name_space: %s\n" % self.DebugFormatString(self.name_space_)) return res - kkey = 1 - _TEXT = ( - "ErrorCode", - "key", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + kkey = 1 + kname_space = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "key", + 2: "name_space", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -189,8 +225,9 @@ class MemcacheGetResponse_Item(ProtocolBuffer.ProtocolMessage): self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -201,8 +238,9 @@ class MemcacheGetResponse_Item(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -213,8 +251,9 @@ class MemcacheGetResponse_Item(ProtocolBuffer.ProtocolMessage): self.flags_ = x def clear_flags(self): - self.has_flags_ = 0 - self.flags_ = 0 + if self.has_flags_: + self.has_flags_ = 0 + self.flags_ = 0 def has_flags(self): return self.has_flags_ @@ -369,30 +408,30 @@ class MemcacheGetResponse(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kItemGroup = 1 kItemkey = 2 kItemvalue = 3 kItemflags = 4 - _TEXT = ( - "ErrorCode", - "Item", - "key", - "value", - "flags", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.FLOAT, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Item", + 2: "key", + 3: "value", + 4: "flags", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STARTGROUP, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.FLOAT, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -418,8 +457,9 @@ class MemcacheSetRequest_Item(ProtocolBuffer.ProtocolMessage): self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -430,8 +470,9 @@ class MemcacheSetRequest_Item(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -442,8 +483,9 @@ class MemcacheSetRequest_Item(ProtocolBuffer.ProtocolMessage): self.flags_ = x def clear_flags(self): - self.has_flags_ = 0 - self.flags_ = 0 + if self.has_flags_: + self.has_flags_ = 0 + self.flags_ = 0 def has_flags(self): return self.has_flags_ @@ -454,8 +496,9 @@ class MemcacheSetRequest_Item(ProtocolBuffer.ProtocolMessage): self.set_policy_ = x def clear_set_policy(self): - self.has_set_policy_ = 0 - self.set_policy_ = 1 + if self.has_set_policy_: + self.has_set_policy_ = 0 + self.set_policy_ = 1 def has_set_policy(self): return self.has_set_policy_ @@ -466,8 +509,9 @@ class MemcacheSetRequest_Item(ProtocolBuffer.ProtocolMessage): self.expiration_time_ = x def clear_expiration_time(self): - self.has_expiration_time_ = 0 - self.expiration_time_ = 0 + if self.has_expiration_time_: + self.has_expiration_time_ = 0 + self.expiration_time_ = 0 def has_expiration_time(self): return self.has_expiration_time_ @@ -584,6 +628,8 @@ class MemcacheSetRequest(ProtocolBuffer.ProtocolMessage): def SetPolicy_Name(cls, x): return cls._SetPolicy_NAMES.get(x, "") SetPolicy_Name = classmethod(SetPolicy_Name) + has_name_space_ = 0 + name_space_ = "" def __init__(self, contents=None): self.item_ = [] @@ -605,16 +651,32 @@ class MemcacheSetRequest(ProtocolBuffer.ProtocolMessage): def clear_item(self): self.item_ = [] + def name_space(self): return self.name_space_ + + def set_name_space(self, x): + self.has_name_space_ = 1 + self.name_space_ = x + + def clear_name_space(self): + if self.has_name_space_: + self.has_name_space_ = 0 + self.name_space_ = "" + + def has_name_space(self): return self.has_name_space_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.item_size()): self.add_item().CopyFrom(x.item(i)) + if (x.has_name_space()): self.set_name_space(x.name_space()) def Equals(self, x): if x is self: return 1 if len(self.item_) != len(x.item_): return 0 for e1, e2 in zip(self.item_, x.item_): if e1 != e2: return 0 + if self.has_name_space_ != x.has_name_space_: return 0 + if self.has_name_space_ and self.name_space_ != x.name_space_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -627,16 +689,21 @@ class MemcacheSetRequest(ProtocolBuffer.ProtocolMessage): n = 0 n += 2 * len(self.item_) for i in xrange(len(self.item_)): n += self.item_[i].ByteSize() + if (self.has_name_space_): n += 1 + self.lengthString(len(self.name_space_)) return n + 0 def Clear(self): self.clear_item() + self.clear_name_space() def OutputUnchecked(self, out): for i in xrange(len(self.item_)): out.putVarInt32(11) self.item_[i].OutputUnchecked(out) out.putVarInt32(12) + if (self.has_name_space_): + out.putVarInt32(58) + out.putPrefixedString(self.name_space_) def TryMerge(self, d): while d.avail() > 0: @@ -644,6 +711,9 @@ class MemcacheSetRequest(ProtocolBuffer.ProtocolMessage): if tt == 11: self.add_item().TryMerge(d) continue + if tt == 58: + self.set_name_space(d.getPrefixedString()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -658,40 +728,42 @@ class MemcacheSetRequest(ProtocolBuffer.ProtocolMessage): res+=e.__str__(prefix + " ", printElemNumber) res+=prefix+"}\n" cnt+=1 + if self.has_name_space_: res+=prefix+("name_space: %s\n" % self.DebugFormatString(self.name_space_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kItemGroup = 1 kItemkey = 2 kItemvalue = 3 kItemflags = 4 kItemset_policy = 5 kItemexpiration_time = 6 - - _TEXT = ( - "ErrorCode", - "Item", - "key", - "value", - "flags", - "set_policy", - "expiration_time", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.FLOAT, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.FLOAT, - - ) + kname_space = 7 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Item", + 2: "key", + 3: "value", + 4: "flags", + 5: "set_policy", + 6: "expiration_time", + 7: "name_space", + }, 7) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STARTGROUP, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.FLOAT, + 5: ProtocolBuffer.Encoder.NUMERIC, + 6: ProtocolBuffer.Encoder.FLOAT, + 7: ProtocolBuffer.Encoder.STRING, + }, 7, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -780,18 +852,21 @@ class MemcacheSetResponse(ProtocolBuffer.ProtocolMessage): cnt+=1 return res - kset_status = 1 - _TEXT = ( - "ErrorCode", - "set_status", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + kset_status = 1 - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "set_status", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -811,8 +886,9 @@ class MemcacheDeleteRequest_Item(ProtocolBuffer.ProtocolMessage): self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -823,8 +899,9 @@ class MemcacheDeleteRequest_Item(ProtocolBuffer.ProtocolMessage): self.delete_time_ = x def clear_delete_time(self): - self.has_delete_time_ = 0 - self.delete_time_ = 0 + if self.has_delete_time_: + self.has_delete_time_ = 0 + self.delete_time_ = 0 def has_delete_time(self): return self.has_delete_time_ @@ -888,6 +965,8 @@ class MemcacheDeleteRequest_Item(ProtocolBuffer.ProtocolMessage): return res class MemcacheDeleteRequest(ProtocolBuffer.ProtocolMessage): + has_name_space_ = 0 + name_space_ = "" def __init__(self, contents=None): self.item_ = [] @@ -909,16 +988,32 @@ class MemcacheDeleteRequest(ProtocolBuffer.ProtocolMessage): def clear_item(self): self.item_ = [] + def name_space(self): return self.name_space_ + + def set_name_space(self, x): + self.has_name_space_ = 1 + self.name_space_ = x + + def clear_name_space(self): + if self.has_name_space_: + self.has_name_space_ = 0 + self.name_space_ = "" + + def has_name_space(self): return self.has_name_space_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.item_size()): self.add_item().CopyFrom(x.item(i)) + if (x.has_name_space()): self.set_name_space(x.name_space()) def Equals(self, x): if x is self: return 1 if len(self.item_) != len(x.item_): return 0 for e1, e2 in zip(self.item_, x.item_): if e1 != e2: return 0 + if self.has_name_space_ != x.has_name_space_: return 0 + if self.has_name_space_ and self.name_space_ != x.name_space_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -931,16 +1026,21 @@ class MemcacheDeleteRequest(ProtocolBuffer.ProtocolMessage): n = 0 n += 2 * len(self.item_) for i in xrange(len(self.item_)): n += self.item_[i].ByteSize() + if (self.has_name_space_): n += 1 + self.lengthString(len(self.name_space_)) return n + 0 def Clear(self): self.clear_item() + self.clear_name_space() def OutputUnchecked(self, out): for i in xrange(len(self.item_)): out.putVarInt32(11) self.item_[i].OutputUnchecked(out) out.putVarInt32(12) + if (self.has_name_space_): + out.putVarInt32(34) + out.putPrefixedString(self.name_space_) def TryMerge(self, d): while d.avail() > 0: @@ -948,6 +1048,9 @@ class MemcacheDeleteRequest(ProtocolBuffer.ProtocolMessage): if tt == 11: self.add_item().TryMerge(d) continue + if tt == 34: + self.set_name_space(d.getPrefixedString()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -962,28 +1065,33 @@ class MemcacheDeleteRequest(ProtocolBuffer.ProtocolMessage): res+=e.__str__(prefix + " ", printElemNumber) res+=prefix+"}\n" cnt+=1 + if self.has_name_space_: res+=prefix+("name_space: %s\n" % self.DebugFormatString(self.name_space_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kItemGroup = 1 kItemkey = 2 kItemdelete_time = 3 - - _TEXT = ( - "ErrorCode", - "Item", - "key", - "delete_time", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.FLOAT, - - ) + kname_space = 4 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Item", + 2: "key", + 3: "delete_time", + 4: "name_space", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STARTGROUP, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.FLOAT, + 4: ProtocolBuffer.Encoder.STRING, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1070,18 +1178,21 @@ class MemcacheDeleteResponse(ProtocolBuffer.ProtocolMessage): cnt+=1 return res - kdelete_status = 1 - _TEXT = ( - "ErrorCode", - "delete_status", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + kdelete_status = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "delete_status", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1100,10 +1211,14 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): has_key_ = 0 key_ = "" + has_name_space_ = 0 + name_space_ = "" has_delta_ = 0 delta_ = 1 has_direction_ = 0 direction_ = 1 + has_initial_value_ = 0 + initial_value_ = 0 def __init__(self, contents=None): if contents is not None: self.MergeFromString(contents) @@ -1115,11 +1230,25 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ + def name_space(self): return self.name_space_ + + def set_name_space(self, x): + self.has_name_space_ = 1 + self.name_space_ = x + + def clear_name_space(self): + if self.has_name_space_: + self.has_name_space_ = 0 + self.name_space_ = "" + + def has_name_space(self): return self.has_name_space_ + def delta(self): return self.delta_ def set_delta(self, x): @@ -1127,8 +1256,9 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): self.delta_ = x def clear_delta(self): - self.has_delta_ = 0 - self.delta_ = 1 + if self.has_delta_: + self.has_delta_ = 0 + self.delta_ = 1 def has_delta(self): return self.has_delta_ @@ -1139,26 +1269,46 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): self.direction_ = x def clear_direction(self): - self.has_direction_ = 0 - self.direction_ = 1 + if self.has_direction_: + self.has_direction_ = 0 + self.direction_ = 1 def has_direction(self): return self.has_direction_ + def initial_value(self): return self.initial_value_ + + def set_initial_value(self, x): + self.has_initial_value_ = 1 + self.initial_value_ = x + + def clear_initial_value(self): + if self.has_initial_value_: + self.has_initial_value_ = 0 + self.initial_value_ = 0 + + def has_initial_value(self): return self.has_initial_value_ + def MergeFrom(self, x): assert x is not self if (x.has_key()): self.set_key(x.key()) + if (x.has_name_space()): self.set_name_space(x.name_space()) if (x.has_delta()): self.set_delta(x.delta()) if (x.has_direction()): self.set_direction(x.direction()) + if (x.has_initial_value()): self.set_initial_value(x.initial_value()) def Equals(self, x): if x is self: return 1 if self.has_key_ != x.has_key_: return 0 if self.has_key_ and self.key_ != x.key_: return 0 + if self.has_name_space_ != x.has_name_space_: return 0 + if self.has_name_space_ and self.name_space_ != x.name_space_: return 0 if self.has_delta_ != x.has_delta_: return 0 if self.has_delta_ and self.delta_ != x.delta_: return 0 if self.has_direction_ != x.has_direction_: return 0 if self.has_direction_ and self.direction_ != x.direction_: return 0 + if self.has_initial_value_ != x.has_initial_value_: return 0 + if self.has_initial_value_ and self.initial_value_ != x.initial_value_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -1172,14 +1322,18 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): def ByteSize(self): n = 0 n += self.lengthString(len(self.key_)) + if (self.has_name_space_): n += 1 + self.lengthString(len(self.name_space_)) if (self.has_delta_): n += 1 + self.lengthVarInt64(self.delta_) if (self.has_direction_): n += 1 + self.lengthVarInt64(self.direction_) + if (self.has_initial_value_): n += 1 + self.lengthVarInt64(self.initial_value_) return n + 1 def Clear(self): self.clear_key() + self.clear_name_space() self.clear_delta() self.clear_direction() + self.clear_initial_value() def OutputUnchecked(self, out): out.putVarInt32(10) @@ -1190,6 +1344,12 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): if (self.has_direction_): out.putVarInt32(24) out.putVarInt32(self.direction_) + if (self.has_name_space_): + out.putVarInt32(34) + out.putPrefixedString(self.name_space_) + if (self.has_initial_value_): + out.putVarInt32(40) + out.putVarUint64(self.initial_value_) def TryMerge(self, d): while d.avail() > 0: @@ -1203,6 +1363,12 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): if tt == 24: self.set_direction(d.getVarInt32()) continue + if tt == 34: + self.set_name_space(d.getPrefixedString()) + continue + if tt == 40: + self.set_initial_value(d.getVarUint64()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -1210,30 +1376,39 @@ class MemcacheIncrementRequest(ProtocolBuffer.ProtocolMessage): def __str__(self, prefix="", printElemNumber=0): res="" if self.has_key_: res+=prefix+("key: %s\n" % self.DebugFormatString(self.key_)) + if self.has_name_space_: res+=prefix+("name_space: %s\n" % self.DebugFormatString(self.name_space_)) if self.has_delta_: res+=prefix+("delta: %s\n" % self.DebugFormatInt64(self.delta_)) if self.has_direction_: res+=prefix+("direction: %s\n" % self.DebugFormatInt32(self.direction_)) + if self.has_initial_value_: res+=prefix+("initial_value: %s\n" % self.DebugFormatInt64(self.initial_value_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kkey = 1 + kname_space = 4 kdelta = 2 kdirection = 3 - - _TEXT = ( - "ErrorCode", - "key", - "delta", - "direction", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + kinitial_value = 5 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "key", + 2: "delta", + 3: "direction", + 4: "name_space", + 5: "initial_value", + }, 5) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.STRING, + 5: ProtocolBuffer.Encoder.NUMERIC, + }, 5, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1251,8 +1426,9 @@ class MemcacheIncrementResponse(ProtocolBuffer.ProtocolMessage): self.new_value_ = x def clear_new_value(self): - self.has_new_value_ = 0 - self.new_value_ = 0 + if self.has_new_value_: + self.has_new_value_ = 0 + self.new_value_ = 0 def has_new_value(self): return self.has_new_value_ @@ -1299,18 +1475,21 @@ class MemcacheIncrementResponse(ProtocolBuffer.ProtocolMessage): if self.has_new_value_: res+=prefix+("new_value: %s\n" % self.DebugFormatInt64(self.new_value_)) return res - knew_value = 1 - _TEXT = ( - "ErrorCode", - "new_value", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + knew_value = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "new_value", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1354,13 +1533,17 @@ class MemcacheFlushRequest(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1404,13 +1587,17 @@ class MemcacheFlushResponse(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1454,13 +1641,17 @@ class MemcacheStatsRequest(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1488,8 +1679,9 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): self.hits_ = x def clear_hits(self): - self.has_hits_ = 0 - self.hits_ = 0 + if self.has_hits_: + self.has_hits_ = 0 + self.hits_ = 0 def has_hits(self): return self.has_hits_ @@ -1500,8 +1692,9 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): self.misses_ = x def clear_misses(self): - self.has_misses_ = 0 - self.misses_ = 0 + if self.has_misses_: + self.has_misses_ = 0 + self.misses_ = 0 def has_misses(self): return self.has_misses_ @@ -1512,8 +1705,9 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): self.byte_hits_ = x def clear_byte_hits(self): - self.has_byte_hits_ = 0 - self.byte_hits_ = 0 + if self.has_byte_hits_: + self.has_byte_hits_ = 0 + self.byte_hits_ = 0 def has_byte_hits(self): return self.has_byte_hits_ @@ -1524,8 +1718,9 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): self.items_ = x def clear_items(self): - self.has_items_ = 0 - self.items_ = 0 + if self.has_items_: + self.has_items_ = 0 + self.items_ = 0 def has_items(self): return self.has_items_ @@ -1536,8 +1731,9 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): self.bytes_ = x def clear_bytes(self): - self.has_bytes_ = 0 - self.bytes_ = 0 + if self.has_bytes_: + self.has_bytes_ = 0 + self.bytes_ = 0 def has_bytes(self): return self.has_bytes_ @@ -1548,8 +1744,9 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): self.oldest_item_age_ = x def clear_oldest_item_age(self): - self.has_oldest_item_age_ = 0 - self.oldest_item_age_ = 0 + if self.has_oldest_item_age_: + self.has_oldest_item_age_ = 0 + self.oldest_item_age_ = 0 def has_oldest_item_age(self): return self.has_oldest_item_age_ @@ -1673,6 +1870,10 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): if self.has_oldest_item_age_: res+=prefix+("oldest_item_age: %s\n" % self.DebugFormatFixed32(self.oldest_item_age_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + khits = 1 kmisses = 2 kbyte_hits = 3 @@ -1680,31 +1881,25 @@ class MergedNamespaceStats(ProtocolBuffer.ProtocolMessage): kbytes = 5 koldest_item_age = 6 - _TEXT = ( - "ErrorCode", - "hits", - "misses", - "byte_hits", - "items", - "bytes", - "oldest_item_age", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.FLOAT, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "hits", + 2: "misses", + 3: "byte_hits", + 4: "items", + 5: "bytes", + 6: "oldest_item_age", + }, 6) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.NUMERIC, + 6: ProtocolBuffer.Encoder.FLOAT, + }, 6, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1728,8 +1923,9 @@ class MemcacheStatsResponse(ProtocolBuffer.ProtocolMessage): def mutable_stats(self): self.has_stats_ = 1; return self.stats() def clear_stats(self): - self.has_stats_ = 0; - if self.stats_ is not None: self.stats_.Clear() + if self.has_stats_: + self.has_stats_ = 0; + if self.stats_ is not None: self.stats_.Clear() def has_stats(self): return self.has_stats_ @@ -1784,18 +1980,21 @@ class MemcacheStatsResponse(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res - kstats = 1 - _TEXT = ( - "ErrorCode", - "stats", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kstats = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "stats", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" google_appengine/google/appengine/api/memcache/memcache_service_pb.py @@ -119,23 +119,27 @@ class MemcacheServiceStub(apiproxy_stub.APIProxyStub): self._byte_hits = 0 self._cache_creation_time = self._gettime() - def _GetKey(self, key): + def _GetKey(self, namespace, key): """Retrieves a CacheEntry from the cache if it hasn't expired. Does not take deletion timeout into account. Args: + namespace: The namespace that keys are stored under. key: The key to retrieve from the cache. Returns: The corresponding CacheEntry instance, or None if it was not found or has already expired. """ - entry = self._the_cache.get(key, None) + namespace_dict = self._the_cache.get(namespace, None) + if namespace_dict is None: + return None + entry = namespace_dict.get(key, None) if entry is None: return None elif entry.CheckExpired(): - del self._the_cache[key] + del namespace_dict[key] return None else: return entry @@ -147,9 +151,10 @@ class MemcacheServiceStub(apiproxy_stub.APIProxyStub): request: A MemcacheGetRequest. response: A MemcacheGetResponse. """ + namespace = request.name_space() keys = set(request.key_list()) for key in keys: - entry = self._GetKey(key) + entry = self._GetKey(namespace, key) if entry is None or entry.CheckLocked(): self._misses += 1 continue @@ -167,10 +172,11 @@ class MemcacheServiceStub(apiproxy_stub.APIProxyStub): request: A MemcacheSetRequest. response: A MemcacheSetResponse. """ + namespace = request.name_space() for item in request.item_list(): key = item.key() set_policy = item.set_policy() - old_entry = self._GetKey(key) + old_entry = self._GetKey(namespace, key) set_status = MemcacheSetResponse.NOT_STORED if ((set_policy == MemcacheSetRequest.SET) or @@ -180,10 +186,12 @@ class MemcacheServiceStub(apiproxy_stub.APIProxyStub): if (old_entry is None or set_policy == MemcacheSetRequest.SET or not old_entry.CheckLocked()): - self._the_cache[key] = CacheEntry(item.value(), - item.expiration_time(), - item.flags(), - gettime=self._gettime) + if namespace not in self._the_cache: + self._the_cache[namespace] = {} + self._the_cache[namespace][key] = CacheEntry(item.value(), + item.expiration_time(), + item.flags(), + gettime=self._gettime) set_status = MemcacheSetResponse.STORED response.add_set_status(set_status) @@ -195,15 +203,16 @@ class MemcacheServiceStub(apiproxy_stub.APIProxyStub): request: A MemcacheDeleteRequest. response: A MemcacheDeleteResponse. """ + namespace = request.name_space() for item in request.item_list(): key = item.key() - entry = self._GetKey(key) + entry = self._GetKey(namespace, key) delete_status = MemcacheDeleteResponse.DELETED if entry is None: delete_status = MemcacheDeleteResponse.NOT_FOUND elif item.delete_time() == 0: - del self._the_cache[key] + del self._the_cache[namespace][key] else: entry.ExpireAndLock(item.delete_time()) @@ -216,16 +225,28 @@ class MemcacheServiceStub(apiproxy_stub.APIProxyStub): request: A MemcacheIncrementRequest. response: A MemcacheIncrementResponse. """ + namespace = request.name_space() key = request.key() - entry = self._GetKey(key) + entry = self._GetKey(namespace, key) if entry is None: - return + if not request.has_initial_value(): + return + if namespace not in self._the_cache: + self._the_cache[namespace] = {} + self._the_cache[namespace][key] = CacheEntry(str(request.initial_value()), + expiration=0, + flags=0, + gettime=self._gettime) + entry = self._GetKey(namespace, key) + assert entry is not None try: old_value = long(entry.value) - except ValueError, e: + if old_value < 0: + raise ValueError + except ValueError: logging.error('Increment/decrement failed: Could not interpret ' - 'value for key = "%s" as an integer.', key) + 'value for key = "%s" as an unsigned integer.', key) return delta = request.delta() @@ -260,11 +281,13 @@ class MemcacheServiceStub(apiproxy_stub.APIProxyStub): stats.set_hits(self._hits) stats.set_misses(self._misses) stats.set_byte_hits(self._byte_hits) - stats.set_items(len(self._the_cache)) - + items = 0 total_bytes = 0 - for key, entry in self._the_cache.iteritems(): - total_bytes += len(entry.value) + for namespace in self._the_cache.itervalues(): + items += len(namespace) + for entry in namespace.itervalues(): + total_bytes += len(entry.value) + stats.set_items(items) stats.set_bytes(total_bytes) stats.set_oldest_item_age(self._gettime() - self._cache_creation_time) google_appengine/google/appengine/api/memcache/memcache_stub.py @@ -186,16 +186,32 @@ def _is_fetching_self(url, method): return False -def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False, - follow_redirects=True): +def create_rpc(deadline=None, callback=None): + """Creates an RPC object for use with the urlfetch API. + + Args: + deadline: Optional deadline in seconds for the operation; the default + is a system-specific deadline (typically 5 seconds). + callback: Optional callable to invoke on completion. + + Returns: + An apiproxy_stub_map.UserRPC object specialized for this service. + """ + return apiproxy_stub_map.UserRPC('urlfetch', deadline, callback) + + +def fetch(url, payload=None, method=GET, headers={}, + allow_truncated=False, follow_redirects=True, + deadline=None): """Fetches the given HTTP URL, blocking until the result is returned. Other optional parameters are: method: GET, POST, HEAD, PUT, or DELETE - payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE) + payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE). + this is ignored if the method is not POST or PUT. headers: dictionary of HTTP headers to send with the request allow_truncated: if true, truncate large responses and return them without - error. otherwise, ResponseTooLargeError will be thrown when a response is + error. Otherwise, ResponseTooLargeError is raised when a response is truncated. follow_redirects: if true (the default), redirects are transparently followed and the response (if less than 5 @@ -204,6 +220,7 @@ def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False, information. If false, you see the HTTP response yourself, including the 'Location' header, and redirects are not followed. + deadline: deadline in seconds for the operation. We use a HTTP/1.1 compliant proxy to fetch the result. @@ -218,6 +235,20 @@ def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False, of the returned structure, so HTTP errors like 404 do not result in an exception. """ + rpc = create_rpc(deadline=deadline) + make_fetch_call(rpc, url, payload, method, headers, + allow_truncated, follow_redirects) + return rpc.get_result() + + +def make_fetch_call(rpc, url, payload=None, method=GET, headers={}, + allow_truncated=False, follow_redirects=True): + """Executes the RPC call to fetch a given HTTP URL. + + The first argument is a UserRPC instance. See urlfetch.fetch for a + thorough description of remaining arguments. + """ + assert rpc.service == 'urlfetch', repr(rpc.service) if isinstance(method, basestring): method = method.upper() method = _URL_STRING_MAP.get(method, method) @@ -253,38 +284,74 @@ def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False, request.set_followredirects(follow_redirects) + if rpc.deadline is not None: + request.set_deadline(rpc.deadline) + + rpc.make_call('Fetch', request, response, _get_fetch_result, allow_truncated) + + +def _get_fetch_result(rpc): + """Check success, handle exceptions, and return converted RPC result. + + This method waits for the RPC if it has not yet finished, and calls the + post-call hooks on the first invocation. + + Args: + rpc: A UserRPC object. + + Raises: + InvalidURLError if the url was invalid. + DownloadError if there was a problem fetching the url. + ResponseTooLargeError if the response was either truncated (and + allow_truncated=False was passed to make_fetch_call()), or if it + was too big for us to download. + + Returns: + A _URLFetchResult object. + """ + assert rpc.service == 'urlfetch', repr(rpc.service) + assert rpc.method == 'Fetch', repr(rpc.method) try: - apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response) - except apiproxy_errors.ApplicationError, e: - if (e.application_error == + rpc.check_success() + except apiproxy_errors.ApplicationError, err: + if (err.application_error == urlfetch_service_pb.URLFetchServiceError.INVALID_URL): - raise InvalidURLError(str(e)) - if (e.application_error == + raise InvalidURLError(str(err)) + if (err.application_error == urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR): - raise DownloadError(str(e)) - if (e.application_error == + raise DownloadError(str(err)) + if (err.application_error == urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR): - raise DownloadError(str(e)) - if (e.application_error == + raise DownloadError(str(err)) + if (err.application_error == urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE): raise ResponseTooLargeError(None) - if (e.application_error == + if (err.application_error == urlfetch_service_pb.URLFetchServiceError.DEADLINE_EXCEEDED): - raise DownloadError(str(e)) - raise e - result = _URLFetchResult(response) + raise DownloadError(str(err)) + raise err - if not allow_truncated and response.contentwastruncated(): + response = rpc.response + allow_truncated = rpc.user_data + result = _URLFetchResult(response) + if response.contentwastruncated() and not allow_truncated: raise ResponseTooLargeError(result) - return result + Fetch = fetch class _URLFetchResult(object): - """A Pythonic representation of our fetch response protocol buffer.""" + """A Pythonic representation of our fetch response protocol buffer. + """ + def __init__(self, response_proto): + """Constructor. + + Args: + response_proto: the URLFetchResponse proto buffer to wrap. + """ self.__pb = response_proto self.content = response_proto.content() self.status_code = response_proto.statuscode() google_appengine/google/appengine/api/urlfetch.py @@ -22,7 +22,6 @@ import dummy_thread as thread __pychecker__ = """maxreturns=0 maxbranches=0 no-callinit unusednames=printElemNumber,debug_strs no-special""" -from google.appengine.api.api_base_pb import StringProto class URLFetchServiceError(ProtocolBuffer.ProtocolMessage): OK = 0 @@ -83,13 +82,17 @@ class URLFetchServiceError(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -109,8 +112,9 @@ class URLFetchRequest_Header(ProtocolBuffer.ProtocolMessage): self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -121,8 +125,9 @@ class URLFetchRequest_Header(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -215,6 +220,8 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): payload_ = "" has_followredirects_ = 0 followredirects_ = 1 + has_deadline_ = 0 + deadline_ = 0.0 def __init__(self, contents=None): self.header_ = [] @@ -227,8 +234,9 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): self.method_ = x def clear_method(self): - self.has_method_ = 0 - self.method_ = 0 + if self.has_method_: + self.has_method_ = 0 + self.method_ = 0 def has_method(self): return self.has_method_ @@ -239,8 +247,9 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): self.url_ = x def clear_url(self): - self.has_url_ = 0 - self.url_ = "" + if self.has_url_: + self.has_url_ = 0 + self.url_ = "" def has_url(self): return self.has_url_ @@ -267,8 +276,9 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): self.payload_ = x def clear_payload(self): - self.has_payload_ = 0 - self.payload_ = "" + if self.has_payload_: + self.has_payload_ = 0 + self.payload_ = "" def has_payload(self): return self.has_payload_ @@ -279,11 +289,25 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): self.followredirects_ = x def clear_followredirects(self): - self.has_followredirects_ = 0 - self.followredirects_ = 1 + if self.has_followredirects_: + self.has_followredirects_ = 0 + self.followredirects_ = 1 def has_followredirects(self): return self.has_followredirects_ + def deadline(self): return self.deadline_ + + def set_deadline(self, x): + self.has_deadline_ = 1 + self.deadline_ = x + + def clear_deadline(self): + if self.has_deadline_: + self.has_deadline_ = 0 + self.deadline_ = 0.0 + + def has_deadline(self): return self.has_deadline_ + def MergeFrom(self, x): assert x is not self @@ -292,6 +316,7 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): for i in xrange(x.header_size()): self.add_header().CopyFrom(x.header(i)) if (x.has_payload()): self.set_payload(x.payload()) if (x.has_followredirects()): self.set_followredirects(x.followredirects()) + if (x.has_deadline()): self.set_deadline(x.deadline()) def Equals(self, x): if x is self: return 1 @@ -306,6 +331,8 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): if self.has_payload_ and self.payload_ != x.payload_: return 0 if self.has_followredirects_ != x.has_followredirects_: return 0 if self.has_followredirects_ and self.followredirects_ != x.followredirects_: return 0 + if self.has_deadline_ != x.has_deadline_: return 0 + if self.has_deadline_ and self.deadline_ != x.deadline_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -330,6 +357,7 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): for i in xrange(len(self.header_)): n += self.header_[i].ByteSize() if (self.has_payload_): n += 1 + self.lengthString(len(self.payload_)) if (self.has_followredirects_): n += 2 + if (self.has_deadline_): n += 9 return n + 2 def Clear(self): @@ -338,6 +366,7 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): self.clear_header() self.clear_payload() self.clear_followredirects() + self.clear_deadline() def OutputUnchecked(self, out): out.putVarInt32(8) @@ -354,6 +383,9 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): if (self.has_followredirects_): out.putVarInt32(56) out.putBoolean(self.followredirects_) + if (self.has_deadline_): + out.putVarInt32(65) + out.putDouble(self.deadline_) def TryMerge(self, d): while d.avail() > 0: @@ -373,6 +405,9 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): if tt == 56: self.set_followredirects(d.getBoolean()) continue + if tt == 65: + self.set_deadline(d.getDouble()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -391,8 +426,13 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): cnt+=1 if self.has_payload_: res+=prefix+("Payload: %s\n" % self.DebugFormatString(self.payload_)) if self.has_followredirects_: res+=prefix+("FollowRedirects: %s\n" % self.DebugFormatBool(self.followredirects_)) + if self.has_deadline_: res+=prefix+("Deadline: %s\n" % self.DebugFormat(self.deadline_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kMethod = 1 kUrl = 2 kHeaderGroup = 3 @@ -400,35 +440,31 @@ class URLFetchRequest(ProtocolBuffer.ProtocolMessage): kHeaderValue = 5 kPayload = 6 kFollowRedirects = 7 - - _TEXT = ( - "ErrorCode", - "Method", - "Url", - "Header", - "Key", - "Value", - "Payload", - "FollowRedirects", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + kDeadline = 8 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Method", + 2: "Url", + 3: "Header", + 4: "Key", + 5: "Value", + 6: "Payload", + 7: "FollowRedirects", + 8: "Deadline", + }, 8) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STARTGROUP, + 4: ProtocolBuffer.Encoder.STRING, + 5: ProtocolBuffer.Encoder.STRING, + 6: ProtocolBuffer.Encoder.STRING, + 7: ProtocolBuffer.Encoder.NUMERIC, + 8: ProtocolBuffer.Encoder.DOUBLE, + }, 8, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -448,8 +484,9 @@ class URLFetchResponse_Header(ProtocolBuffer.ProtocolMessage): self.key_ = x def clear_key(self): - self.has_key_ = 0 - self.key_ = "" + if self.has_key_: + self.has_key_ = 0 + self.key_ = "" def has_key(self): return self.has_key_ @@ -460,8 +497,9 @@ class URLFetchResponse_Header(ProtocolBuffer.ProtocolMessage): self.value_ = x def clear_value(self): - self.has_value_ = 0 - self.value_ = "" + if self.has_value_: + self.has_value_ = 0 + self.value_ = "" def has_value(self): return self.has_value_ @@ -534,6 +572,10 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): statuscode_ = 0 has_contentwastruncated_ = 0 contentwastruncated_ = 0 + has_externalbytessent_ = 0 + externalbytessent_ = 0 + has_externalbytesreceived_ = 0 + externalbytesreceived_ = 0 def __init__(self, contents=None): self.header_ = [] @@ -546,8 +588,9 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): self.content_ = x def clear_content(self): - self.has_content_ = 0 - self.content_ = "" + if self.has_content_: + self.has_content_ = 0 + self.content_ = "" def has_content(self): return self.has_content_ @@ -558,8 +601,9 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): self.statuscode_ = x def clear_statuscode(self): - self.has_statuscode_ = 0 - self.statuscode_ = 0 + if self.has_statuscode_: + self.has_statuscode_ = 0 + self.statuscode_ = 0 def has_statuscode(self): return self.has_statuscode_ @@ -586,11 +630,38 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): self.contentwastruncated_ = x def clear_contentwastruncated(self): - self.has_contentwastruncated_ = 0 - self.contentwastruncated_ = 0 + if self.has_contentwastruncated_: + self.has_contentwastruncated_ = 0 + self.contentwastruncated_ = 0 def has_contentwastruncated(self): return self.has_contentwastruncated_ + def externalbytessent(self): return self.externalbytessent_ + + def set_externalbytessent(self, x): + self.has_externalbytessent_ = 1 + self.externalbytessent_ = x + + def clear_externalbytessent(self): + if self.has_externalbytessent_: + self.has_externalbytessent_ = 0 + self.externalbytessent_ = 0 + + def has_externalbytessent(self): return self.has_externalbytessent_ + + def externalbytesreceived(self): return self.externalbytesreceived_ + + def set_externalbytesreceived(self, x): + self.has_externalbytesreceived_ = 1 + self.externalbytesreceived_ = x + + def clear_externalbytesreceived(self): + if self.has_externalbytesreceived_: + self.has_externalbytesreceived_ = 0 + self.externalbytesreceived_ = 0 + + def has_externalbytesreceived(self): return self.has_externalbytesreceived_ + def MergeFrom(self, x): assert x is not self @@ -598,6 +669,8 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): if (x.has_statuscode()): self.set_statuscode(x.statuscode()) for i in xrange(x.header_size()): self.add_header().CopyFrom(x.header(i)) if (x.has_contentwastruncated()): self.set_contentwastruncated(x.contentwastruncated()) + if (x.has_externalbytessent()): self.set_externalbytessent(x.externalbytessent()) + if (x.has_externalbytesreceived()): self.set_externalbytesreceived(x.externalbytesreceived()) def Equals(self, x): if x is self: return 1 @@ -610,6 +683,10 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): if e1 != e2: return 0 if self.has_contentwastruncated_ != x.has_contentwastruncated_: return 0 if self.has_contentwastruncated_ and self.contentwastruncated_ != x.contentwastruncated_: return 0 + if self.has_externalbytessent_ != x.has_externalbytessent_: return 0 + if self.has_externalbytessent_ and self.externalbytessent_ != x.externalbytessent_: return 0 + if self.has_externalbytesreceived_ != x.has_externalbytesreceived_: return 0 + if self.has_externalbytesreceived_ and self.externalbytesreceived_ != x.externalbytesreceived_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -629,6 +706,8 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): n += 2 * len(self.header_) for i in xrange(len(self.header_)): n += self.header_[i].ByteSize() if (self.has_contentwastruncated_): n += 2 + if (self.has_externalbytessent_): n += 1 + self.lengthVarInt64(self.externalbytessent_) + if (self.has_externalbytesreceived_): n += 1 + self.lengthVarInt64(self.externalbytesreceived_) return n + 1 def Clear(self): @@ -636,6 +715,8 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): self.clear_statuscode() self.clear_header() self.clear_contentwastruncated() + self.clear_externalbytessent() + self.clear_externalbytesreceived() def OutputUnchecked(self, out): if (self.has_content_): @@ -650,6 +731,12 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): if (self.has_contentwastruncated_): out.putVarInt32(48) out.putBoolean(self.contentwastruncated_) + if (self.has_externalbytessent_): + out.putVarInt32(56) + out.putVarInt64(self.externalbytessent_) + if (self.has_externalbytesreceived_): + out.putVarInt32(64) + out.putVarInt64(self.externalbytesreceived_) def TryMerge(self, d): while d.avail() > 0: @@ -666,6 +753,12 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): if tt == 48: self.set_contentwastruncated(d.getBoolean()) continue + if tt == 56: + self.set_externalbytessent(d.getVarInt64()) + continue + if tt == 64: + self.set_externalbytesreceived(d.getVarInt64()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -683,40 +776,46 @@ class URLFetchResponse(ProtocolBuffer.ProtocolMessage): res+=prefix+"}\n" cnt+=1 if self.has_contentwastruncated_: res+=prefix+("ContentWasTruncated: %s\n" % self.DebugFormatBool(self.contentwastruncated_)) + if self.has_externalbytessent_: res+=prefix+("ExternalBytesSent: %s\n" % self.DebugFormatInt64(self.externalbytessent_)) + if self.has_externalbytesreceived_: res+=prefix+("ExternalBytesReceived: %s\n" % self.DebugFormatInt64(self.externalbytesreceived_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kContent = 1 kStatusCode = 2 kHeaderGroup = 3 kHeaderKey = 4 kHeaderValue = 5 kContentWasTruncated = 6 - - _TEXT = ( - "ErrorCode", - "Content", - "StatusCode", - "Header", - "Key", - "Value", - "ContentWasTruncated", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + kExternalBytesSent = 7 + kExternalBytesReceived = 8 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Content", + 2: "StatusCode", + 3: "Header", + 4: "Key", + 5: "Value", + 6: "ContentWasTruncated", + 7: "ExternalBytesSent", + 8: "ExternalBytesReceived", + }, 8) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.STARTGROUP, + 4: ProtocolBuffer.Encoder.STRING, + 5: ProtocolBuffer.Encoder.STRING, + 6: ProtocolBuffer.Encoder.NUMERIC, + 7: ProtocolBuffer.Encoder.NUMERIC, + 8: ProtocolBuffer.Encoder.NUMERIC, + }, 8, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" google_appengine/google/appengine/api/urlfetch_service_pb.py @@ -19,9 +19,11 @@ +import gzip import httplib import logging import socket +import StringIO import urllib import urlparse @@ -51,11 +53,8 @@ _API_CALL_DEADLINE = 5.0 _UNTRUSTED_REQUEST_HEADERS = frozenset([ - 'accept-encoding', 'content-length', 'host', - 'referer', - 'user-agent', 'vary', 'via', 'x-forwarded-for', @@ -104,17 +103,26 @@ class URLFetchServiceStub(apiproxy_stub.APIProxyStub): raise apiproxy_errors.ApplicationError( urlfetch_service_pb.URLFetchServiceError.INVALID_URL) + if not host: + logging.error('Missing host.') + raise apiproxy_errors.ApplicationError( + urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR) + sanitized_headers = self._SanitizeHttpHeaders(_UNTRUSTED_REQUEST_HEADERS, request.header_list()) request.clear_header() request.header_list().extend(sanitized_headers) + deadline = _API_CALL_DEADLINE + if request.has_deadline(): + deadline = request.deadline() self._RetrieveURL(request.url(), payload, method, request.header_list(), response, - follow_redirects=request.followredirects()) + follow_redirects=request.followredirects(), + deadline=deadline) def _RetrieveURL(self, url, payload, method, headers, response, - follow_redirects=True): + follow_redirects=True, deadline=_API_CALL_DEADLINE): """Retrieves a URL. Args: @@ -125,6 +133,7 @@ class URLFetchServiceStub(apiproxy_stub.APIProxyStub): response: Response object follow_redirects: optional setting (defaulting to True) for whether or not we should transparently follow redirects (up to MAX_REDIRECTS) + deadline: Number of seconds to wait for the urlfetch to finish. Raises: Raises an apiproxy_errors.ApplicationError exception with FETCH_ERROR @@ -146,13 +155,20 @@ class URLFetchServiceStub(apiproxy_stub.APIProxyStub): 'urlfetch received %s ; port %s is not allowed in production!' % (url, port)) - if host == '' and protocol == '': + if protocol and not host: + logging.error('Missing host on redirect; target url is %s' % url) + raise apiproxy_errors.ApplicationError( + urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR) + + if not host and not protocol: host = last_host protocol = last_protocol adjusted_headers = { - 'Host': host, - 'Accept': '*/*', + 'User-Agent': + 'AppEngine-Google; (+http://code.google.com/appengine)', + 'Host': host, + 'Accept-Encoding': 'gzip', } if payload is not None: adjusted_headers['Content-Length'] = len(payload) @@ -160,7 +176,12 @@ class URLFetchServiceStub(apiproxy_stub.APIProxyStub): adjusted_headers['Content-Type'] = 'application/x-www-form-urlencoded' for header in headers: - adjusted_headers[header.key().title()] = header.value() + if header.key().title().lower() == 'user-agent': + adjusted_headers['User-Agent'] = ( + '%s %s' % + (header.value(), adjusted_headers['User-Agent'])) + else: + adjusted_headers[header.key().title()] = header.value() logging.debug('Making HTTP request: host = %s, ' 'url = %s, payload = %s, headers = %s', @@ -186,10 +207,13 @@ class URLFetchServiceStub(apiproxy_stub.APIProxyStub): orig_timeout = socket.getdefaulttimeout() try: - socket.setdefaulttimeout(_API_CALL_DEADLINE) + socket.setdefaulttimeout(deadline) connection.request(method, full_path, payload, adjusted_headers) http_response = connection.getresponse() - http_response_data = http_response.read() + if method == 'HEAD': + http_response_data = '' + else: + http_response_data = http_response.read() finally: socket.setdefaulttimeout(orig_timeout) connection.close() @@ -206,8 +230,17 @@ class URLFetchServiceStub(apiproxy_stub.APIProxyStub): urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg) else: response.set_statuscode(http_response.status) + if http_response.getheader('content-encoding') == 'gzip': + gzip_stream = StringIO.StringIO(http_response_data) + gzip_file = gzip.GzipFile(fileobj=gzip_stream) + http_response_data = gzip_file.read() response.set_content(http_response_data[:MAX_RESPONSE_SIZE]) for header_key, header_value in http_response.getheaders(): + if (header_key.lower() == 'content-encoding' and + header_value == 'gzip'): + continue + if header_key.lower() == 'content-length': + header_value = str(len(response.content())) header_proto = response.add_header() header_proto.set_key(header_key) header_proto.set_value(header_value) @@ -229,4 +262,9 @@ class URLFetchServiceStub(apiproxy_stub.APIProxyStub): untrusted_headers: set of untrusted headers names headers: list of string pairs, first is header name and the second is header's value """ + prohibited_headers = [h.key() for h in headers + if h.key().lower() in untrusted_headers] + if prohibited_headers: + logging.warn('Stripped prohibited headers from URLFetch request: %s', + prohibited_headers) return (h for h in headers if h.key().lower() not in untrusted_headers) google_appengine/google/appengine/api/urlfetch_stub.py @@ -22,15 +22,23 @@ import dummy_thread as thread __pychecker__ = """maxreturns=0 maxbranches=0 no-callinit unusednames=printElemNumber,debug_strs no-special""" -from google.appengine.api.api_base_pb import StringProto +from google.appengine.api.api_base_pb import * class UserServiceError(ProtocolBuffer.ProtocolMessage): OK = 0 REDIRECT_URL_TOO_LONG = 1 + NOT_ALLOWED = 2 + OAUTH_INVALID_TOKEN = 3 + OAUTH_INVALID_REQUEST = 4 + OAUTH_ERROR = 5 _ErrorCode_NAMES = { 0: "OK", 1: "REDIRECT_URL_TOO_LONG", + 2: "NOT_ALLOWED", + 3: "OAUTH_INVALID_TOKEN", + 4: "OAUTH_INVALID_REQUEST", + 5: "OAUTH_ERROR", } def ErrorCode_Name(cls, x): return cls._ErrorCode_NAMES.get(x, "") @@ -75,15 +83,685 @@ class UserServiceError(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CreateLoginURLRequest(ProtocolBuffer.ProtocolMessage): + has_destination_url_ = 0 + destination_url_ = "" + has_auth_domain_ = 0 + auth_domain_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def destination_url(self): return self.destination_url_ + + def set_destination_url(self, x): + self.has_destination_url_ = 1 + self.destination_url_ = x + + def clear_destination_url(self): + if self.has_destination_url_: + self.has_destination_url_ = 0 + self.destination_url_ = "" + + def has_destination_url(self): return self.has_destination_url_ + + def auth_domain(self): return self.auth_domain_ + + def set_auth_domain(self, x): + self.has_auth_domain_ = 1 + self.auth_domain_ = x + + def clear_auth_domain(self): + if self.has_auth_domain_: + self.has_auth_domain_ = 0 + self.auth_domain_ = "" + + def has_auth_domain(self): return self.has_auth_domain_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_destination_url()): self.set_destination_url(x.destination_url()) + if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain()) + + def Equals(self, x): + if x is self: return 1 + if self.has_destination_url_ != x.has_destination_url_: return 0 + if self.has_destination_url_ and self.destination_url_ != x.destination_url_: return 0 + if self.has_auth_domain_ != x.has_auth_domain_: return 0 + if self.has_auth_domain_ and self.auth_domain_ != x.auth_domain_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_destination_url_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: destination_url not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(len(self.destination_url_)) + if (self.has_auth_domain_): n += 1 + self.lengthString(len(self.auth_domain_)) + return n + 1 + + def Clear(self): + self.clear_destination_url() + self.clear_auth_domain() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putPrefixedString(self.destination_url_) + if (self.has_auth_domain_): + out.putVarInt32(18) + out.putPrefixedString(self.auth_domain_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_destination_url(d.getPrefixedString()) + continue + if tt == 18: + self.set_auth_domain(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_destination_url_: res+=prefix+("destination_url: %s\n" % self.DebugFormatString(self.destination_url_)) + if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kdestination_url = 1 + kauth_domain = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "destination_url", + 2: "auth_domain", + }, 2) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CreateLoginURLResponse(ProtocolBuffer.ProtocolMessage): + has_login_url_ = 0 + login_url_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def login_url(self): return self.login_url_ + + def set_login_url(self, x): + self.has_login_url_ = 1 + self.login_url_ = x + + def clear_login_url(self): + if self.has_login_url_: + self.has_login_url_ = 0 + self.login_url_ = "" + + def has_login_url(self): return self.has_login_url_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_login_url()): self.set_login_url(x.login_url()) + + def Equals(self, x): + if x is self: return 1 + if self.has_login_url_ != x.has_login_url_: return 0 + if self.has_login_url_ and self.login_url_ != x.login_url_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_login_url_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: login_url not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(len(self.login_url_)) + return n + 1 + + def Clear(self): + self.clear_login_url() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putPrefixedString(self.login_url_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_login_url(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_login_url_: res+=prefix+("login_url: %s\n" % self.DebugFormatString(self.login_url_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + klogin_url = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "login_url", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CreateLogoutURLRequest(ProtocolBuffer.ProtocolMessage): + has_destination_url_ = 0 + destination_url_ = "" + has_auth_domain_ = 0 + auth_domain_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def destination_url(self): return self.destination_url_ + + def set_destination_url(self, x): + self.has_destination_url_ = 1 + self.destination_url_ = x + + def clear_destination_url(self): + if self.has_destination_url_: + self.has_destination_url_ = 0 + self.destination_url_ = "" + + def has_destination_url(self): return self.has_destination_url_ + + def auth_domain(self): return self.auth_domain_ + + def set_auth_domain(self, x): + self.has_auth_domain_ = 1 + self.auth_domain_ = x + + def clear_auth_domain(self): + if self.has_auth_domain_: + self.has_auth_domain_ = 0 + self.auth_domain_ = "" + + def has_auth_domain(self): return self.has_auth_domain_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_destination_url()): self.set_destination_url(x.destination_url()) + if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain()) + + def Equals(self, x): + if x is self: return 1 + if self.has_destination_url_ != x.has_destination_url_: return 0 + if self.has_destination_url_ and self.destination_url_ != x.destination_url_: return 0 + if self.has_auth_domain_ != x.has_auth_domain_: return 0 + if self.has_auth_domain_ and self.auth_domain_ != x.auth_domain_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_destination_url_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: destination_url not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(len(self.destination_url_)) + if (self.has_auth_domain_): n += 1 + self.lengthString(len(self.auth_domain_)) + return n + 1 + + def Clear(self): + self.clear_destination_url() + self.clear_auth_domain() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putPrefixedString(self.destination_url_) + if (self.has_auth_domain_): + out.putVarInt32(18) + out.putPrefixedString(self.auth_domain_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_destination_url(d.getPrefixedString()) + continue + if tt == 18: + self.set_auth_domain(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_destination_url_: res+=prefix+("destination_url: %s\n" % self.DebugFormatString(self.destination_url_)) + if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kdestination_url = 1 + kauth_domain = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "destination_url", + 2: "auth_domain", + }, 2) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CreateLogoutURLResponse(ProtocolBuffer.ProtocolMessage): + has_logout_url_ = 0 + logout_url_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def logout_url(self): return self.logout_url_ + + def set_logout_url(self, x): + self.has_logout_url_ = 1 + self.logout_url_ = x + + def clear_logout_url(self): + if self.has_logout_url_: + self.has_logout_url_ = 0 + self.logout_url_ = "" + + def has_logout_url(self): return self.has_logout_url_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_logout_url()): self.set_logout_url(x.logout_url()) + + def Equals(self, x): + if x is self: return 1 + if self.has_logout_url_ != x.has_logout_url_: return 0 + if self.has_logout_url_ and self.logout_url_ != x.logout_url_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_logout_url_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: logout_url not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(len(self.logout_url_)) + return n + 1 + + def Clear(self): + self.clear_logout_url() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putPrefixedString(self.logout_url_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_logout_url(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_logout_url_: res+=prefix+("logout_url: %s\n" % self.DebugFormatString(self.logout_url_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + klogout_url = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "logout_url", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class GetOAuthUserRequest(ProtocolBuffer.ProtocolMessage): + + def __init__(self, contents=None): + pass + if contents is not None: self.MergeFromString(contents) + + + def MergeFrom(self, x): + assert x is not self + + def Equals(self, x): + if x is self: return 1 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + return initialized + + def ByteSize(self): + n = 0 + return n + 0 + + def Clear(self): + pass + + def OutputUnchecked(self, out): + pass + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class GetOAuthUserResponse(ProtocolBuffer.ProtocolMessage): + has_email_ = 0 + email_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def email(self): return self.email_ + + def set_email(self, x): + self.has_email_ = 1 + self.email_ = x + + def clear_email(self): + if self.has_email_: + self.has_email_ = 0 + self.email_ = "" + + def has_email(self): return self.has_email_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_email()): self.set_email(x.email()) + + def Equals(self, x): + if x is self: return 1 + if self.has_email_ != x.has_email_: return 0 + if self.has_email_ and self.email_ != x.email_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + return initialized + + def ByteSize(self): + n = 0 + if (self.has_email_): n += 1 + self.lengthString(len(self.email_)) + return n + 0 + + def Clear(self): + self.clear_email() + + def OutputUnchecked(self, out): + if (self.has_email_): + out.putVarInt32(10) + out.putPrefixedString(self.email_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_email(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_email_: res+=prefix+("email: %s\n" % self.DebugFormatString(self.email_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kemail = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "email", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CheckOAuthSignatureRequest(ProtocolBuffer.ProtocolMessage): + + def __init__(self, contents=None): + pass + if contents is not None: self.MergeFromString(contents) + + + def MergeFrom(self, x): + assert x is not self + + def Equals(self, x): + if x is self: return 1 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + return initialized + + def ByteSize(self): + n = 0 + return n + 0 + + def Clear(self): + pass + + def OutputUnchecked(self, out): + pass + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CheckOAuthSignatureResponse(ProtocolBuffer.ProtocolMessage): + has_oauth_consumer_key_ = 0 + oauth_consumer_key_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def oauth_consumer_key(self): return self.oauth_consumer_key_ + + def set_oauth_consumer_key(self, x): + self.has_oauth_consumer_key_ = 1 + self.oauth_consumer_key_ = x + + def clear_oauth_consumer_key(self): + if self.has_oauth_consumer_key_: + self.has_oauth_consumer_key_ = 0 + self.oauth_consumer_key_ = "" + + def has_oauth_consumer_key(self): return self.has_oauth_consumer_key_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_oauth_consumer_key()): self.set_oauth_consumer_key(x.oauth_consumer_key()) + + def Equals(self, x): + if x is self: return 1 + if self.has_oauth_consumer_key_ != x.has_oauth_consumer_key_: return 0 + if self.has_oauth_consumer_key_ and self.oauth_consumer_key_ != x.oauth_consumer_key_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + return initialized + + def ByteSize(self): + n = 0 + if (self.has_oauth_consumer_key_): n += 1 + self.lengthString(len(self.oauth_consumer_key_)) + return n + 0 + + def Clear(self): + self.clear_oauth_consumer_key() + + def OutputUnchecked(self, out): + if (self.has_oauth_consumer_key_): + out.putVarInt32(10) + out.putPrefixedString(self.oauth_consumer_key_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_oauth_consumer_key(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_oauth_consumer_key_: res+=prefix+("oauth_consumer_key: %s\n" % self.DebugFormatString(self.oauth_consumer_key_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + koauth_consumer_key = 1 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "oauth_consumer_key", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" -__all__ = ['UserServiceError'] +__all__ = ['UserServiceError','CreateLoginURLRequest','CreateLoginURLResponse','CreateLogoutURLRequest','CreateLogoutURLResponse','GetOAuthUserRequest','GetOAuthUserResponse','CheckOAuthSignatureRequest','CheckOAuthSignatureResponse'] google_appengine/google/appengine/api/user_service_pb.py @@ -65,9 +65,9 @@ class UserServiceStub(apiproxy_stub.APIProxyStub): response: the login URL; a base.StringProto """ self.__num_requests += 1 - response.set_value( + response.set_login_url( self._login_url % - urllib.quote(self._AddHostToContinueURL(request.value()))) + urllib.quote(self._AddHostToContinueURL(request.destination_url()))) def _Dynamic_CreateLogoutURL(self, request, response): """Trivial implementation of UserService.CreateLogoutURL(). @@ -77,9 +77,9 @@ class UserServiceStub(apiproxy_stub.APIProxyStub): response: the logout URL; a base.StringProto """ self.__num_requests += 1 - response.set_value( + response.set_logout_url( self._logout_url % - urllib.quote(self._AddHostToContinueURL(request.value()))) + urllib.quote(self._AddHostToContinueURL(request.destination_url()))) def _AddHostToContinueURL(self, continue_url): """Adds the request host to the continue url if no host is specified. google_appengine/google/appengine/api/user_service_stub.py @@ -20,9 +20,9 @@ Classes defined here: User: object representing a user. Error: base exception type - BadRequestError: UserService exception UserNotFoundError: UserService exception - BackendError: UserService exception + RedirectTooLongError: UserService exception + NotAllowedError: UserService exception """ @@ -33,7 +33,6 @@ Classes defined here: import os from google.appengine.api import apiproxy_stub_map from google.appengine.api import user_service_pb -from google.appengine.api import api_base_pb from google.appengine.runtime import apiproxy_errors @@ -51,10 +50,15 @@ class RedirectTooLongError(Error): """ +class NotAllowedError(Error): + """Raised by UserService calls if the requested redirect URL is not allowed. + """ + + class User(object): """A user. - We provide here the email address, nickname, and auth domain for a user. + We provide the email address, nickname, auth domain, and id for a user. A nickname is a human-readable string which uniquely identifies a Google user, akin to a username. It will be an email address for some users, but @@ -62,12 +66,18 @@ class User(object): """ - def __init__(self, email=None, _auth_domain=None): + __user_id = None + + def __init__(self, email=None, _auth_domain=None, _user_id=None): """Constructor. Args: - # email is optional. it defaults to the current user. - email: string + email: An optional string of the user's email address. It defaults to + the current user's email address. + + Raises: + UserNotFoundError: Raised if the user is not logged in and the email + argument is empty. """ if _auth_domain is None: _auth_domain = os.environ.get('AUTH_DOMAIN') @@ -79,12 +89,15 @@ class User(object): if email is None: assert 'USER_EMAIL' in os.environ email = os.environ['USER_EMAIL'] + if _user_id is None and 'USER_ID' in os.environ: + _user_id = os.environ['USER_ID'] if not email: raise UserNotFoundError self.__email = email self.__auth_domain = _auth_domain + self.__user_id = _user_id or None def nickname(self): """Return this user's nickname. @@ -104,6 +117,13 @@ class User(object): """Return this user's email address.""" return self.__email + def user_id(self): + """Return either a permanent unique identifying string or None. + + If the email address was set explicity, this will return None. + """ + return self.__user_id + def auth_domain(self): """Return this user's auth domain.""" return self.__auth_domain @@ -115,7 +135,11 @@ class User(object): return str(self.nickname()) def __repr__(self): - return "users.User(email='%s')" % self.email() + if self.__user_id: + return "users.User(email='%s',_user_id='%s')" % (self.email(), + self.user_id()) + else: + return "users.User(email='%s')" % self.email() def __hash__(self): return hash((self.__email, self.__auth_domain)) @@ -127,7 +151,7 @@ class User(object): (other.__email, other.__auth_domain)) -def create_login_url(dest_url): +def create_login_url(dest_url, _auth_domain=None): """Computes the login URL for this request and specified destination URL. Args: @@ -138,23 +162,29 @@ def create_login_url(dest_url): Returns: string """ - req = user_service_pb.StringProto() - resp = user_service_pb.StringProto() - req.set_value(dest_url) + req = user_service_pb.CreateLoginURLRequest() + resp = user_service_pb.CreateLoginURLResponse() + req.set_destination_url(dest_url) + if _auth_domain: + req.set_auth_domain(_auth_domain) + try: apiproxy_stub_map.MakeSyncCall('user', 'CreateLoginURL', req, resp) except apiproxy_errors.ApplicationError, e: if (e.application_error == user_service_pb.UserServiceError.REDIRECT_URL_TOO_LONG): raise RedirectTooLongError + elif (e.application_error == + user_service_pb.UserServiceError.NOT_ALLOWED): + raise NotAllowedError else: raise e - return resp.value() + return resp.login_url() CreateLoginURL = create_login_url -def create_logout_url(dest_url): +def create_logout_url(dest_url, _auth_domain=None): """Computes the logout URL for this request and specified destination URL. Args: @@ -165,9 +195,12 @@ def create_logout_url(dest_url): Returns: string """ - req = user_service_pb.StringProto() - resp = user_service_pb.StringProto() - req.set_value(dest_url) + req = user_service_pb.CreateLogoutURLRequest() + resp = user_service_pb.CreateLogoutURLResponse() + req.set_destination_url(dest_url) + if _auth_domain: + req.set_auth_domain(_auth_domain) + try: apiproxy_stub_map.MakeSyncCall('user', 'CreateLogoutURL', req, resp) except apiproxy_errors.ApplicationError, e: @@ -176,7 +209,7 @@ def create_logout_url(dest_url): raise RedirectTooLongError else: raise e - return resp.value() + return resp.logout_url() CreateLogoutURL = create_logout_url @@ -198,6 +231,6 @@ def is_current_user_admin(): the User class, because admin status is not persisted in the datastore. It only exists for the user making this request right now. """ - return (os.environ.get('USER_IS_ADMIN', '0')) == "1" + return (os.environ.get('USER_IS_ADMIN', '0')) == '1' IsCurrentUserAdmin = is_current_user_admin google_appengine/google/appengine/api/users.py @@ -919,7 +919,9 @@ class Repeated(Validator): 'but found \'%s\'.' % value) for item in value: - if not isinstance(item, self.constructor): + if isinstance(self.constructor, Validator): + item = self.constructor.Validate(item) + elif not isinstance(item, self.constructor): raise ValidationError('Repeated items must be %s, but found \'%s\'.' % (str(self.constructor), str(item))) google_appengine/google/appengine/api/validation.py @@ -59,8 +59,9 @@ class CapabilityConfigList(ProtocolBuffer.ProtocolMessage): def mutable_default_config(self): self.has_default_config_ = 1; return self.default_config() def clear_default_config(self): - self.has_default_config_ = 0; - if self.default_config_ is not None: self.default_config_.Clear() + if self.has_default_config_: + self.has_default_config_ = 0; + if self.default_config_ is not None: self.default_config_.Clear() def has_default_config(self): return self.has_default_config_ @@ -142,22 +143,24 @@ class CapabilityConfigList(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res - kconfig = 1 - kdefault_config = 2 - _TEXT = ( - "ErrorCode", - "config", - "default_config", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + kconfig = 1 + kdefault_config = 2 - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "config", + 2: "default_config", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -203,8 +206,9 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): self.package_ = x def clear_package(self): - self.has_package_ = 0 - self.package_ = "" + if self.has_package_: + self.has_package_ = 0 + self.package_ = "" def has_package(self): return self.has_package_ @@ -215,8 +219,9 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): self.capability_ = x def clear_capability(self): - self.has_capability_ = 0 - self.capability_ = "" + if self.has_capability_: + self.has_capability_ = 0 + self.capability_ = "" def has_capability(self): return self.has_capability_ @@ -227,8 +232,9 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): self.status_ = x def clear_status(self): - self.has_status_ = 0 - self.status_ = 4 + if self.has_status_: + self.has_status_ = 0 + self.status_ = 4 def has_status(self): return self.has_status_ @@ -239,8 +245,9 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): self.scheduled_time_ = x def clear_scheduled_time(self): - self.has_scheduled_time_ = 0 - self.scheduled_time_ = "" + if self.has_scheduled_time_: + self.has_scheduled_time_ = 0 + self.scheduled_time_ = "" def has_scheduled_time(self): return self.has_scheduled_time_ @@ -251,8 +258,9 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): self.internal_message_ = x def clear_internal_message(self): - self.has_internal_message_ = 0 - self.internal_message_ = "" + if self.has_internal_message_: + self.has_internal_message_ = 0 + self.internal_message_ = "" def has_internal_message(self): return self.has_internal_message_ @@ -263,8 +271,9 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): self.admin_message_ = x def clear_admin_message(self): - self.has_admin_message_ = 0 - self.admin_message_ = "" + if self.has_admin_message_: + self.has_admin_message_ = 0 + self.admin_message_ = "" def has_admin_message(self): return self.has_admin_message_ @@ -275,8 +284,9 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): self.error_message_ = x def clear_error_message(self): - self.has_error_message_ = 0 - self.error_message_ = "" + if self.has_error_message_: + self.has_error_message_ = 0 + self.error_message_ = "" def has_error_message(self): return self.has_error_message_ @@ -401,6 +411,10 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): if self.has_error_message_: res+=prefix+("error_message: %s\n" % self.DebugFormatString(self.error_message_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kpackage = 1 kcapability = 2 kstatus = 3 @@ -409,34 +423,27 @@ class CapabilityConfig(ProtocolBuffer.ProtocolMessage): kadmin_message = 5 kerror_message = 6 - _TEXT = ( - "ErrorCode", - "package", - "capability", - "status", - "internal_message", - "admin_message", - "error_message", - "scheduled_time", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "package", + 2: "capability", + 3: "status", + 4: "internal_message", + 5: "admin_message", + 6: "error_message", + 7: "scheduled_time", + }, 7) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.STRING, + 5: ProtocolBuffer.Encoder.STRING, + 6: ProtocolBuffer.Encoder.STRING, + 7: ProtocolBuffer.Encoder.STRING, + }, 7, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" google_appengine/google/appengine/base/capabilities_pb.py @@ -23,39 +23,40 @@ from antlr3.compat import set, frozenset HIDDEN = BaseRecognizer.HIDDEN THIRD=12 -SEPTEMBER=34 +SEPTEMBER=35 FOURTH=13 SECOND=11 -WEDNESDAY=20 -NOVEMBER=36 -SATURDAY=23 -JULY=32 -APRIL=29 +WEDNESDAY=21 +NOVEMBER=37 +SATURDAY=24 +JULY=33 +APRIL=30 DIGITS=8 -OCTOBER=35 -MAY=30 +OCTOBER=36 +MAY=31 EVERY=6 -FEBRUARY=27 -MONDAY=18 -SUNDAY=24 -JUNE=31 +FEBRUARY=28 +MONDAY=19 +SUNDAY=25 +DAY=18 +JUNE=32 OF=4 -MARCH=28 +MARCH=29 EOF=-1 -JANUARY=26 -MONTH=25 -FRIDAY=22 +JANUARY=27 +MONTH=26 +FRIDAY=23 MINUTES=17 FIFTH=14 TIME=5 -WS=39 -QUARTER=38 -THURSDAY=21 +WS=40 +QUARTER=39 +THURSDAY=22 COMMA=9 -DECEMBER=37 -AUGUST=33 +DECEMBER=38 +AUGUST=34 DIGIT=7 -TUESDAY=19 +TUESDAY=20 HOURS=16 FOURTH_OR_FIFTH=15 FIRST=10 @@ -100,10 +101,10 @@ class GrocLexer(Lexer): if LA1 == 48: LA1_1 = self.input.LA(2) - if ((48 <= LA1_1 <= 57)) : - alt1 = 2 - elif (LA1_1 == 58) : + if (LA1_1 == 58) : alt1 = 1 + elif ((48 <= LA1_1 <= 57)) : + alt1 = 2 else: nvae = NoViableAltException("", 1, 1, self.input) @@ -112,10 +113,10 @@ class GrocLexer(Lexer): elif LA1 == 49: LA1_2 = self.input.LA(2) - if ((48 <= LA1_2 <= 57)) : - alt1 = 3 - elif (LA1_2 == 58) : + if (LA1_2 == 58) : alt1 = 1 + elif ((48 <= LA1_2 <= 57)) : + alt1 = 3 else: nvae = NoViableAltException("", 1, 2, self.input) @@ -124,7 +125,7 @@ class GrocLexer(Lexer): elif LA1 == 50: LA1_3 = self.input.LA(2) - if ((48 <= LA1_3 <= 52)) : + if ((48 <= LA1_3 <= 51)) : alt1 = 4 elif (LA1_3 == 58) : alt1 = 1 @@ -169,7 +170,7 @@ class GrocLexer(Lexer): pass pass self.match(50) - self.matchRange(48, 52) + self.matchRange(48, 51) @@ -436,6 +437,27 @@ class GrocLexer(Lexer): + def mDAY(self, ): + + try: + _type = DAY + _channel = DEFAULT_CHANNEL + + pass + self.match("day") + + + + self._state.type = _type + self._state.channel = _channel + + finally: + + pass + + + + def mMONDAY(self, ): try: @@ -1330,7 +1352,7 @@ class GrocLexer(Lexer): def mTokens(self): - alt25 = 36 + alt25 = 37 alt25 = self.dfa25.predict(self.input) if alt25 == 1: pass @@ -1369,146 +1391,151 @@ class GrocLexer(Lexer): elif alt25 == 8: pass - self.mMONDAY() + self.mDAY() elif alt25 == 9: pass - self.mTUESDAY() + self.mMONDAY() elif alt25 == 10: pass - self.mWEDNESDAY() + self.mTUESDAY() elif alt25 == 11: pass - self.mTHURSDAY() + self.mWEDNESDAY() elif alt25 == 12: pass - self.mFRIDAY() + self.mTHURSDAY() elif alt25 == 13: pass - self.mSATURDAY() + self.mFRIDAY() elif alt25 == 14: pass - self.mSUNDAY() + self.mSATURDAY() elif alt25 == 15: pass - self.mJANUARY() + self.mSUNDAY() elif alt25 == 16: pass - self.mFEBRUARY() + self.mJANUARY() elif alt25 == 17: pass - self.mMARCH() + self.mFEBRUARY() elif alt25 == 18: pass - self.mAPRIL() + self.mMARCH() elif alt25 == 19: pass - self.mMAY() + self.mAPRIL() elif alt25 == 20: pass - self.mJUNE() + self.mMAY() elif alt25 == 21: pass - self.mJULY() + self.mJUNE() elif alt25 == 22: pass - self.mAUGUST() + self.mJULY() elif alt25 == 23: pass - self.mSEPTEMBER() + self.mAUGUST() elif alt25 == 24: pass - self.mOCTOBER() + self.mSEPTEMBER() elif alt25 == 25: pass - self.mNOVEMBER() + self.mOCTOBER() elif alt25 == 26: pass - self.mDECEMBER() + self.mNOVEMBER() elif alt25 == 27: pass - self.mMONTH() + self.mDECEMBER() elif alt25 == 28: pass - self.mQUARTER() + self.mMONTH() elif alt25 == 29: pass - self.mEVERY() + self.mQUARTER() elif alt25 == 30: pass - self.mHOURS() + self.mEVERY() elif alt25 == 31: pass - self.mMINUTES() + self.mHOURS() elif alt25 == 32: pass - self.mCOMMA() + self.mMINUTES() elif alt25 == 33: pass - self.mOF() + self.mCOMMA() elif alt25 == 34: pass - self.mWS() + self.mOF() elif alt25 == 35: pass - self.mDIGIT() + self.mWS() elif alt25 == 36: pass + self.mDIGIT() + + + elif alt25 == 37: + pass self.mDIGITS() @@ -1519,73 +1546,74 @@ class GrocLexer(Lexer): DFA25_eot = DFA.unpack( - u"\1\uffff\4\27\2\uffff\1\27\1\uffff\2\27\16\uffff\1\36\1\uffff\2" - u"\36\31\uffff\1\74\6\uffff" + u"\1\uffff\4\30\2\uffff\1\30\1\uffff\2\30\14\uffff\1\36\3\uffff\2" + u"\36\33\uffff\1\76\6\uffff" ) DFA25_eof = DFA.unpack( - u"\75\uffff" + u"\77\uffff" ) DFA25_min = DFA.unpack( - u"\1\11\4\60\1\145\1\141\1\60\1\150\2\60\1\141\1\uffff\1\141\1\160" - u"\1\143\11\uffff\1\72\1\uffff\2\72\3\uffff\1\146\3\uffff\1\143\3" - u"\uffff\1\151\2\uffff\1\156\1\162\2\uffff\1\154\6\uffff\1\164\6" + u"\1\11\4\60\1\145\1\141\1\60\1\150\2\60\2\141\1\uffff\1\141\1\160" + u"\1\143\6\uffff\1\72\3\uffff\2\72\3\uffff\1\146\3\uffff\1\143\3" + u"\uffff\1\151\4\uffff\1\156\1\162\2\uffff\1\154\6\uffff\1\164\6" u"\uffff" ) DFA25_max = DFA.unpack( - u"\1\167\1\72\1\163\1\156\2\162\1\165\1\164\1\165\1\164\1\72\1\157" - u"\1\uffff\2\165\1\146\11\uffff\1\72\1\uffff\2\72\3\uffff\1\162\3" - u"\uffff\1\160\3\uffff\1\165\2\uffff\1\156\1\171\2\uffff\1\156\6" - u"\uffff\1\164\6\uffff" + u"\1\167\1\72\1\163\1\156\2\162\1\165\1\164\1\165\1\164\1\72\1\145" + u"\1\157\1\uffff\2\165\1\146\6\uffff\1\72\3\uffff\2\72\3\uffff\1" + u"\162\3\uffff\1\160\3\uffff\1\165\4\uffff\1\156\1\171\2\uffff\1" + u"\156\6\uffff\1\164\6\uffff" ) DFA25_accept = DFA.unpack( - u"\14\uffff\1\12\3\uffff\1\31\1\32\1\34\1\35\1\36\1\40\1\42\1\43" - u"\1\1\1\uffff\1\2\2\uffff\1\3\1\44\1\4\1\uffff\1\7\1\14\1\20\1\uffff" - u"\1\15\1\16\1\5\1\uffff\1\11\1\6\2\uffff\1\37\1\17\1\uffff\1\22" - u"\1\26\1\30\1\41\1\27\1\13\1\uffff\1\21\1\23\1\24\1\25\1\33\1\10" + u"\15\uffff\1\13\3\uffff\1\32\1\35\1\36\1\37\1\41\1\43\1\uffff\1" + u"\44\1\1\1\2\2\uffff\1\3\1\45\1\4\1\uffff\1\7\1\15\1\21\1\uffff" + u"\1\16\1\17\1\5\1\uffff\1\12\1\6\1\10\1\33\2\uffff\1\40\1\20\1\uffff" + u"\1\23\1\27\1\31\1\42\1\30\1\14\1\uffff\1\22\1\24\1\25\1\26\1\34" + u"\1\11" ) DFA25_special = DFA.unpack( - u"\75\uffff" + u"\77\uffff" ) DFA25_transition = [ DFA.unpack(u"\2\26\2\uffff\1\26\22\uffff\1\26\13\uffff\1\25\3\uffff" - u"\1\1\1\2\1\3\1\4\1\7\1\11\4\12\47\uffff\1\16\2\uffff\1\21\1\23" - u"\1\5\1\uffff\1\24\1\uffff\1\15\2\uffff\1\13\1\20\1\17\1\uffff\1" - u"\22\1\uffff\1\6\1\10\2\uffff\1\14"), - DFA.unpack(u"\12\31\1\30"), - DFA.unpack(u"\12\33\1\30\70\uffff\1\32"), - DFA.unpack(u"\5\34\5\36\1\30\63\uffff\1\35"), - DFA.unpack(u"\12\36\1\30\67\uffff\1\37"), + u"\1\1\1\2\1\3\1\4\1\7\1\11\4\12\47\uffff\1\17\2\uffff\1\13\1\23" + u"\1\5\1\uffff\1\24\1\uffff\1\16\2\uffff\1\14\1\21\1\20\1\uffff\1" + u"\22\1\uffff\1\6\1\10\2\uffff\1\15"), + DFA.unpack(u"\12\27\1\31"), + DFA.unpack(u"\12\33\1\31\70\uffff\1\32"), + DFA.unpack(u"\4\34\6\36\1\31\63\uffff\1\35"), + DFA.unpack(u"\12\36\1\31\67\uffff\1\37"), DFA.unpack(u"\1\43\3\uffff\1\40\5\uffff\1\41\2\uffff\1\42"), DFA.unpack(u"\1\45\3\uffff\1\44\17\uffff\1\46"), - DFA.unpack(u"\12\36\1\30\71\uffff\1\47"), + DFA.unpack(u"\12\36\1\31\71\uffff\1\47"), DFA.unpack(u"\1\50\14\uffff\1\51"), - DFA.unpack(u"\12\36\1\30\71\uffff\1\52"), - DFA.unpack(u"\12\36\1\30"), - DFA.unpack(u"\1\54\7\uffff\1\55\5\uffff\1\53"), - DFA.unpack(u""), - DFA.unpack(u"\1\56\23\uffff\1\57"), - DFA.unpack(u"\1\60\4\uffff\1\61"), - DFA.unpack(u"\1\62\2\uffff\1\63"), + DFA.unpack(u"\12\36\1\31\71\uffff\1\52"), + DFA.unpack(u"\12\36\1\31"), + DFA.unpack(u"\1\53\3\uffff\1\54"), + DFA.unpack(u"\1\56\7\uffff\1\57\5\uffff\1\55"), DFA.unpack(u""), + DFA.unpack(u"\1\60\23\uffff\1\61"), + DFA.unpack(u"\1\62\4\uffff\1\63"), + DFA.unpack(u"\1\64\2\uffff\1\65"), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), + DFA.unpack(u"\1\31"), DFA.unpack(u""), DFA.unpack(u""), - DFA.unpack(u"\1\30"), DFA.unpack(u""), - DFA.unpack(u"\1\30"), - DFA.unpack(u"\1\30"), + DFA.unpack(u"\1\31"), + DFA.unpack(u"\1\31"), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), @@ -1593,25 +1621,27 @@ class GrocLexer(Lexer): DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), - DFA.unpack(u"\1\35\14\uffff\1\64"), + DFA.unpack(u"\1\35\14\uffff\1\66"), + DFA.unpack(u""), + DFA.unpack(u""), DFA.unpack(u""), + DFA.unpack(u"\1\37\13\uffff\1\67"), DFA.unpack(u""), DFA.unpack(u""), - DFA.unpack(u"\1\37\13\uffff\1\65"), DFA.unpack(u""), DFA.unpack(u""), - DFA.unpack(u"\1\66"), - DFA.unpack(u"\1\67\6\uffff\1\70"), + DFA.unpack(u"\1\70"), + DFA.unpack(u"\1\71\6\uffff\1\72"), DFA.unpack(u""), DFA.unpack(u""), - DFA.unpack(u"\1\72\1\uffff\1\71"), + DFA.unpack(u"\1\74\1\uffff\1\73"), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), - DFA.unpack(u"\1\73"), + DFA.unpack(u"\1\75"), DFA.unpack(u""), DFA.unpack(u""), DFA.unpack(u""), google_appengine/google/appengine/cron/GrocLexer.py @@ -32,39 +32,40 @@ numOrdinals = len(allOrdinals) HIDDEN = BaseRecognizer.HIDDEN THIRD=12 -SEPTEMBER=34 +SEPTEMBER=35 FOURTH=13 SECOND=11 -WEDNESDAY=20 -NOVEMBER=36 -SATURDAY=23 -JULY=32 -APRIL=29 +WEDNESDAY=21 +NOVEMBER=37 +SATURDAY=24 +JULY=33 +APRIL=30 DIGITS=8 -OCTOBER=35 -MAY=30 +OCTOBER=36 +MAY=31 EVERY=6 -FEBRUARY=27 -MONDAY=18 -SUNDAY=24 -JUNE=31 -MARCH=28 +FEBRUARY=28 +MONDAY=19 +SUNDAY=25 +JUNE=32 +DAY=18 +MARCH=29 OF=4 EOF=-1 -JANUARY=26 -MONTH=25 -FRIDAY=22 +JANUARY=27 +MONTH=26 +FRIDAY=23 FIFTH=14 MINUTES=17 TIME=5 -WS=39 -QUARTER=38 -THURSDAY=21 +WS=40 +QUARTER=39 +THURSDAY=22 COMMA=9 -DECEMBER=37 -AUGUST=33 +DECEMBER=38 +AUGUST=34 DIGIT=7 -TUESDAY=19 +TUESDAY=20 HOURS=16 FIRST=10 FOURTH_OR_FIFTH=15 @@ -72,10 +73,10 @@ FOURTH_OR_FIFTH=15 tokenNames = [ "<invalid>", "<EOR>", "<DOWN>", "<UP>", "OF", "TIME", "EVERY", "DIGIT", "DIGITS", "COMMA", "FIRST", "SECOND", - "THIRD", "FOURTH", "FIFTH", "FOURTH_OR_FIFTH", "HOURS", "MINUTES", "MONDAY", - "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY", "SUNDAY", - "MONTH", "JANUARY", "FEBRUARY", "MARCH", "APRIL", "MAY", "JUNE", "JULY", - "AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER", "QUARTER", + "THIRD", "FOURTH", "FIFTH", "FOURTH_OR_FIFTH", "HOURS", "MINUTES", "DAY", + "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY", + "SUNDAY", "MONTH", "JANUARY", "FEBRUARY", "MARCH", "APRIL", "MAY", "JUNE", + "JULY", "AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER", "QUARTER", "WS" ] @@ -95,6 +96,17 @@ class GrocParser(Parser): Parser.__init__(self, input, state) + self.dfa3 = self.DFA3( + self, 3, + eot = self.DFA3_eot, + eof = self.DFA3_eof, + min = self.DFA3_min, + max = self.DFA3_max, + accept = self.DFA3_accept, + special = self.DFA3_special, + transition = self.DFA3_transition + ) + @@ -160,7 +172,7 @@ class GrocParser(Parser): if ((DIGIT <= LA1_1 <= DIGITS)) : alt1 = 2 - elif ((MONDAY <= LA1_1 <= SUNDAY)) : + elif ((DAY <= LA1_1 <= SUNDAY)) : alt1 = 1 else: nvae = NoViableAltException("", 1, 1, self.input) @@ -214,57 +226,77 @@ class GrocParser(Parser): try: pass pass - pass - pass - pass - self._state.following.append(self.FOLLOW_ordinals_in_specifictime69) - self.ordinals() + alt3 = 2 + alt3 = self.dfa3.predict(self.input) + if alt3 == 1: + pass + pass + pass + self._state.following.append(self.FOLLOW_ordinals_in_specifictime69) + self.ordinals() - self._state.following.pop() - self._state.following.append(self.FOLLOW_weekdays_in_specifictime71) - self.weekdays() + self._state.following.pop() + self._state.following.append(self.FOLLOW_weekdays_in_specifictime71) + self.weekdays() - self._state.following.pop() + self._state.following.pop() + self.match(self.input, OF, self.FOLLOW_OF_in_specifictime74) + alt2 = 2 + LA2_0 = self.input.LA(1) + if ((MONTH <= LA2_0 <= DECEMBER)) : + alt2 = 1 + elif ((FIRST <= LA2_0 <= THIRD) or LA2_0 == QUARTER) : + alt2 = 2 + else: + nvae = NoViableAltException("", 2, 0, self.input) + raise nvae - self.match(self.input, OF, self.FOLLOW_OF_in_specifictime75) - alt2 = 2 - LA2_0 = self.input.LA(1) + if alt2 == 1: + pass + self._state.following.append(self.FOLLOW_monthspec_in_specifictime77) + self.monthspec() - if ((MONTH <= LA2_0 <= DECEMBER)) : - alt2 = 1 - elif ((FIRST <= LA2_0 <= THIRD) or LA2_0 == QUARTER) : - alt2 = 2 - else: - nvae = NoViableAltException("", 2, 0, self.input) + self._state.following.pop() - raise nvae - if alt2 == 1: - pass - self._state.following.append(self.FOLLOW_monthspec_in_specifictime78) - self.monthspec() + elif alt2 == 2: + pass + self._state.following.append(self.FOLLOW_quarterspec_in_specifictime79) + self.quarterspec() + + self._state.following.pop() + - self._state.following.pop() - elif alt2 == 2: + + + + + elif alt3 == 2: + pass pass - self._state.following.append(self.FOLLOW_quarterspec_in_specifictime80) - self.quarterspec() + self._state.following.append(self.FOLLOW_ordinals_in_specifictime96) + self.ordinals() + + self._state.following.pop() + self._state.following.append(self.FOLLOW_weekdays_in_specifictime98) + self.weekdays() self._state.following.pop() + self.month_set = set(range(1,13)) - TIME1=self.match(self.input, TIME, self.FOLLOW_TIME_in_specifictime93) + TIME1=self.match(self.input, TIME, self.FOLLOW_TIME_in_specifictime112) self.time_string = TIME1.text @@ -294,7 +326,7 @@ class GrocParser(Parser): try: pass pass - self.match(self.input, EVERY, self.FOLLOW_EVERY_in_interval112) + self.match(self.input, EVERY, self.FOLLOW_EVERY_in_interval131) intervalnum = self.input.LT(1) if (DIGIT <= self.input.LA(1) <= DIGITS): self.input.consume() @@ -308,7 +340,7 @@ class GrocParser(Parser): self.interval_mins = int(intervalnum.text) - self._state.following.append(self.FOLLOW_period_in_interval138) + self._state.following.append(self.FOLLOW_period_in_interval157) period2 = self.period() self._state.following.pop() @@ -341,43 +373,43 @@ class GrocParser(Parser): try: try: pass - alt4 = 2 - LA4_0 = self.input.LA(1) + alt5 = 2 + LA5_0 = self.input.LA(1) - if (LA4_0 == EVERY) : - alt4 = 1 - elif ((FIRST <= LA4_0 <= FOURTH_OR_FIFTH)) : - alt4 = 2 + if (LA5_0 == EVERY) : + alt5 = 1 + elif ((FIRST <= LA5_0 <= FOURTH_OR_FIFTH)) : + alt5 = 2 else: - nvae = NoViableAltException("", 4, 0, self.input) + nvae = NoViableAltException("", 5, 0, self.input) raise nvae - if alt4 == 1: + if alt5 == 1: pass - self.match(self.input, EVERY, self.FOLLOW_EVERY_in_ordinals157) + self.match(self.input, EVERY, self.FOLLOW_EVERY_in_ordinals176) self.ordinal_set = self.ordinal_set.union(allOrdinals) - elif alt4 == 2: + elif alt5 == 2: pass pass - self._state.following.append(self.FOLLOW_ordinal_in_ordinals173) + self._state.following.append(self.FOLLOW_ordinal_in_ordinals192) self.ordinal() self._state.following.pop() while True: - alt3 = 2 - LA3_0 = self.input.LA(1) + alt4 = 2 + LA4_0 = self.input.LA(1) - if (LA3_0 == COMMA) : - alt3 = 1 + if (LA4_0 == COMMA) : + alt4 = 1 - if alt3 == 1: + if alt4 == 1: pass - self.match(self.input, COMMA, self.FOLLOW_COMMA_in_ordinals176) - self._state.following.append(self.FOLLOW_ordinal_in_ordinals178) + self.match(self.input, COMMA, self.FOLLOW_COMMA_in_ordinals195) + self._state.following.append(self.FOLLOW_ordinal_in_ordinals197) self.ordinal() self._state.following.pop() @@ -489,30 +521,58 @@ class GrocParser(Parser): try: try: pass - pass - self._state.following.append(self.FOLLOW_weekday_in_weekdays261) - self.weekday() + alt7 = 2 + LA7_0 = self.input.LA(1) - self._state.following.pop() - while True: - alt5 = 2 - LA5_0 = self.input.LA(1) + if (LA7_0 == DAY) : + alt7 = 1 + elif ((MONDAY <= LA7_0 <= SUNDAY)) : + alt7 = 2 + else: + nvae = NoViableAltException("", 7, 0, self.input) - if (LA5_0 == COMMA) : - alt5 = 1 + raise nvae + if alt7 == 1: + pass + self.match(self.input, DAY, self.FOLLOW_DAY_in_weekdays280) - if alt5 == 1: - pass - self.match(self.input, COMMA, self.FOLLOW_COMMA_in_weekdays264) - self._state.following.append(self.FOLLOW_weekday_in_weekdays266) - self.weekday() + self.weekday_set = set([self.ValueOf(SUNDAY), self.ValueOf(MONDAY), + self.ValueOf(TUESDAY), self.ValueOf(WEDNESDAY), + self.ValueOf(THURSDAY), self.ValueOf(FRIDAY), + self.ValueOf(SATURDAY), self.ValueOf(SUNDAY)]) + + + + elif alt7 == 2: + pass + pass + self._state.following.append(self.FOLLOW_weekday_in_weekdays288) + self.weekday() + + self._state.following.pop() + while True: + alt6 = 2 + LA6_0 = self.input.LA(1) + + if (LA6_0 == COMMA) : + alt6 = 1 + + + if alt6 == 1: + pass + self.match(self.input, COMMA, self.FOLLOW_COMMA_in_weekdays291) + self._state.following.append(self.FOLLOW_weekday_in_weekdays293) + self.weekday() + + self._state.following.pop() + + + else: + break - self._state.following.pop() - else: - break @@ -573,21 +633,21 @@ class GrocParser(Parser): try: try: pass - alt6 = 2 - LA6_0 = self.input.LA(1) + alt8 = 2 + LA8_0 = self.input.LA(1) - if (LA6_0 == MONTH) : - alt6 = 1 - elif ((JANUARY <= LA6_0 <= DECEMBER)) : - alt6 = 2 + if (LA8_0 == MONTH) : + alt8 = 1 + elif ((JANUARY <= LA8_0 <= DECEMBER)) : + alt8 = 2 else: - nvae = NoViableAltException("", 6, 0, self.input) + nvae = NoViableAltException("", 8, 0, self.input) raise nvae - if alt6 == 1: + if alt8 == 1: pass - self.match(self.input, MONTH, self.FOLLOW_MONTH_in_monthspec344) + self.match(self.input, MONTH, self.FOLLOW_MONTH_in_monthspec373) self.month_set = self.month_set.union(set([ self.ValueOf(JANUARY), self.ValueOf(FEBRUARY), self.ValueOf(MARCH), @@ -598,9 +658,9 @@ class GrocParser(Parser): - elif alt6 == 2: + elif alt8 == 2: pass - self._state.following.append(self.FOLLOW_months_in_monthspec354) + self._state.following.append(self.FOLLOW_months_in_monthspec383) self.months() self._state.following.pop() @@ -628,22 +688,22 @@ class GrocParser(Parser): try: pass pass - self._state.following.append(self.FOLLOW_month_in_months371) + self._state.following.append(self.FOLLOW_month_in_months400) self.month() self._state.following.pop() while True: - alt7 = 2 - LA7_0 = self.input.LA(1) + alt9 = 2 + LA9_0 = self.input.LA(1) - if (LA7_0 == COMMA) : - alt7 = 1 + if (LA9_0 == COMMA) : + alt9 = 1 - if alt7 == 1: + if alt9 == 1: pass - self.match(self.input, COMMA, self.FOLLOW_COMMA_in_months374) - self._state.following.append(self.FOLLOW_month_in_months376) + self.match(self.input, COMMA, self.FOLLOW_COMMA_in_months403) + self._state.following.append(self.FOLLOW_month_in_months405) self.month() self._state.following.pop() @@ -709,37 +769,37 @@ class GrocParser(Parser): try: try: pass - alt8 = 2 - LA8_0 = self.input.LA(1) + alt10 = 2 + LA10_0 = self.input.LA(1) - if (LA8_0 == QUARTER) : - alt8 = 1 - elif ((FIRST <= LA8_0 <= THIRD)) : - alt8 = 2 + if (LA10_0 == QUARTER) : + alt10 = 1 + elif ((FIRST <= LA10_0 <= THIRD)) : + alt10 = 2 else: - nvae = NoViableAltException("", 8, 0, self.input) + nvae = NoViableAltException("", 10, 0, self.input) raise nvae - if alt8 == 1: + if alt10 == 1: pass - self.match(self.input, QUARTER, self.FOLLOW_QUARTER_in_quarterspec468) + self.match(self.input, QUARTER, self.FOLLOW_QUARTER_in_quarterspec497) self.month_set = self.month_set.union(set([ self.ValueOf(JANUARY), self.ValueOf(APRIL), self.ValueOf(JULY), self.ValueOf(OCTOBER)])) - elif alt8 == 2: + elif alt10 == 2: pass pass - self._state.following.append(self.FOLLOW_quarter_ordinals_in_quarterspec480) + self._state.following.append(self.FOLLOW_quarter_ordinals_in_quarterspec509) self.quarter_ordinals() self._state.following.pop() - self.match(self.input, MONTH, self.FOLLOW_MONTH_in_quarterspec482) - self.match(self.input, OF, self.FOLLOW_OF_in_quarterspec484) - self.match(self.input, QUARTER, self.FOLLOW_QUARTER_in_quarterspec486) + self.match(self.input, MONTH, self.FOLLOW_MONTH_in_quarterspec511) + self.match(self.input, OF, self.FOLLOW_OF_in_quarterspec513) + self.match(self.input, QUARTER, self.FOLLOW_QUARTER_in_quarterspec515) @@ -767,22 +827,22 @@ class GrocParser(Parser): try: pass pass - self._state.following.append(self.FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals505) + self._state.following.append(self.FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals534) self.month_of_quarter_ordinal() self._state.following.pop() while True: - alt9 = 2 - LA9_0 = self.input.LA(1) + alt11 = 2 + LA11_0 = self.input.LA(1) - if (LA9_0 == COMMA) : - alt9 = 1 + if (LA11_0 == COMMA) : + alt11 = 1 - if alt9 == 1: + if alt11 == 1: pass - self.match(self.input, COMMA, self.FOLLOW_COMMA_in_quarter_ordinals508) - self._state.following.append(self.FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals510) + self.match(self.input, COMMA, self.FOLLOW_COMMA_in_quarter_ordinals537) + self._state.following.append(self.FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals539) self.month_of_quarter_ordinal() self._state.following.pop() @@ -850,43 +910,88 @@ class GrocParser(Parser): + DFA3_eot = DFA.unpack( + u"\13\uffff" + ) + + DFA3_eof = DFA.unpack( + u"\13\uffff" + ) + + DFA3_min = DFA.unpack( + u"\1\6\1\22\1\11\2\4\1\12\2\uffff\1\23\1\11\1\4" + ) + + DFA3_max = DFA.unpack( + u"\1\17\2\31\1\5\1\11\1\17\2\uffff\2\31\1\11" + ) + + DFA3_accept = DFA.unpack( + u"\6\uffff\1\1\1\2\3\uffff" + ) + + DFA3_special = DFA.unpack( + u"\13\uffff" + ) + + + DFA3_transition = [ + DFA.unpack(u"\1\1\3\uffff\6\2"), + DFA.unpack(u"\1\3\7\4"), + DFA.unpack(u"\1\5\10\uffff\1\3\7\4"), + DFA.unpack(u"\1\6\1\7"), + DFA.unpack(u"\1\6\1\7\3\uffff\1\10"), + DFA.unpack(u"\6\11"), + DFA.unpack(u""), + DFA.unpack(u""), + DFA.unpack(u"\7\12"), + DFA.unpack(u"\1\5\10\uffff\1\3\7\4"), + DFA.unpack(u"\1\6\1\7\3\uffff\1\10") + ] + + + DFA3 = DFA + FOLLOW_specifictime_in_timespec44 = frozenset([1]) FOLLOW_interval_in_timespec48 = frozenset([1]) - FOLLOW_ordinals_in_specifictime69 = frozenset([18, 19, 20, 21, 22, 23, 24]) + FOLLOW_ordinals_in_specifictime69 = frozenset([18, 19, 20, 21, 22, 23, 24, 25]) FOLLOW_weekdays_in_specifictime71 = frozenset([4]) - FOLLOW_OF_in_specifictime75 = frozenset([10, 11, 12, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38]) - FOLLOW_monthspec_in_specifictime78 = frozenset([5]) - FOLLOW_quarterspec_in_specifictime80 = frozenset([5]) - FOLLOW_TIME_in_specifictime93 = frozenset([1]) - FOLLOW_EVERY_in_interval112 = frozenset([7, 8]) - FOLLOW_set_in_interval122 = frozenset([16, 17]) - FOLLOW_period_in_interval138 = frozenset([1]) - FOLLOW_EVERY_in_ordinals157 = frozenset([1]) - FOLLOW_ordinal_in_ordinals173 = frozenset([1, 9]) - FOLLOW_COMMA_in_ordinals176 = frozenset([10, 11, 12, 13, 14, 15]) - FOLLOW_ordinal_in_ordinals178 = frozenset([1, 9]) - FOLLOW_set_in_ordinal199 = frozenset([1]) - FOLLOW_set_in_period238 = frozenset([1]) - FOLLOW_weekday_in_weekdays261 = frozenset([1, 9]) - FOLLOW_COMMA_in_weekdays264 = frozenset([18, 19, 20, 21, 22, 23, 24]) - FOLLOW_weekday_in_weekdays266 = frozenset([1, 9]) - FOLLOW_set_in_weekday285 = frozenset([1]) - FOLLOW_MONTH_in_monthspec344 = frozenset([1]) - FOLLOW_months_in_monthspec354 = frozenset([1]) - FOLLOW_month_in_months371 = frozenset([1, 9]) - FOLLOW_COMMA_in_months374 = frozenset([25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37]) - FOLLOW_month_in_months376 = frozenset([1, 9]) - FOLLOW_set_in_month395 = frozenset([1]) - FOLLOW_QUARTER_in_quarterspec468 = frozenset([1]) - FOLLOW_quarter_ordinals_in_quarterspec480 = frozenset([25]) - FOLLOW_MONTH_in_quarterspec482 = frozenset([4]) - FOLLOW_OF_in_quarterspec484 = frozenset([38]) - FOLLOW_QUARTER_in_quarterspec486 = frozenset([1]) - FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals505 = frozenset([1, 9]) - FOLLOW_COMMA_in_quarter_ordinals508 = frozenset([10, 11, 12, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38]) - FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals510 = frozenset([1, 9]) - FOLLOW_set_in_month_of_quarter_ordinal529 = frozenset([1]) + FOLLOW_OF_in_specifictime74 = frozenset([10, 11, 12, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]) + FOLLOW_monthspec_in_specifictime77 = frozenset([5]) + FOLLOW_quarterspec_in_specifictime79 = frozenset([5]) + FOLLOW_ordinals_in_specifictime96 = frozenset([18, 19, 20, 21, 22, 23, 24, 25]) + FOLLOW_weekdays_in_specifictime98 = frozenset([5]) + FOLLOW_TIME_in_specifictime112 = frozenset([1]) + FOLLOW_EVERY_in_interval131 = frozenset([7, 8]) + FOLLOW_set_in_interval141 = frozenset([16, 17]) + FOLLOW_period_in_interval157 = frozenset([1]) + FOLLOW_EVERY_in_ordinals176 = frozenset([1]) + FOLLOW_ordinal_in_ordinals192 = frozenset([1, 9]) + FOLLOW_COMMA_in_ordinals195 = frozenset([10, 11, 12, 13, 14, 15]) + FOLLOW_ordinal_in_ordinals197 = frozenset([1, 9]) + FOLLOW_set_in_ordinal218 = frozenset([1]) + FOLLOW_set_in_period257 = frozenset([1]) + FOLLOW_DAY_in_weekdays280 = frozenset([1]) + FOLLOW_weekday_in_weekdays288 = frozenset([1, 9]) + FOLLOW_COMMA_in_weekdays291 = frozenset([18, 19, 20, 21, 22, 23, 24, 25]) + FOLLOW_weekday_in_weekdays293 = frozenset([1, 9]) + FOLLOW_set_in_weekday314 = frozenset([1]) + FOLLOW_MONTH_in_monthspec373 = frozenset([1]) + FOLLOW_months_in_monthspec383 = frozenset([1]) + FOLLOW_month_in_months400 = frozenset([1, 9]) + FOLLOW_COMMA_in_months403 = frozenset([26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38]) + FOLLOW_month_in_months405 = frozenset([1, 9]) + FOLLOW_set_in_month424 = frozenset([1]) + FOLLOW_QUARTER_in_quarterspec497 = frozenset([1]) + FOLLOW_quarter_ordinals_in_quarterspec509 = frozenset([26]) + FOLLOW_MONTH_in_quarterspec511 = frozenset([4]) + FOLLOW_OF_in_quarterspec513 = frozenset([39]) + FOLLOW_QUARTER_in_quarterspec515 = frozenset([1]) + FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals534 = frozenset([1, 9]) + FOLLOW_COMMA_in_quarter_ordinals537 = frozenset([10, 11, 12, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]) + FOLLOW_month_of_quarter_ordinal_in_quarter_ordinals539 = frozenset([1, 9]) + FOLLOW_set_in_month_of_quarter_ordinal558 = frozenset([1]) google_appengine/google/appengine/cron/GrocParser.py @@ -22,25 +22,38 @@ A Groc schedule looks like '1st,2nd monday 9:00', or 'every 20 mins'. This module takes a parsed schedule (produced by Antlr) and creates objects that can produce times that match this schedule. -A parsed schedule is one of two types - an Interval, and a Specific Time. +A parsed schedule is one of two types - an Interval or a Specific Time. See the class docstrings for more. Extensions to be considered: allowing a comma separated list of times to run allowing the user to specify particular days of the month to run - """ import calendar import datetime +try: + import pytz +except ImportError: + pytz = None + import groc HOURS = 'hours' MINUTES = 'minutes' +try: + from pytz import NonExistentTimeError + from pytz import AmbiguousTimeError +except ImportError: + class NonExistentTimeError(Exception): + pass + class AmbiguousTimeError(Exception): + pass + def GrocTimeSpecification(schedule): """Factory function. @@ -53,15 +66,17 @@ def GrocTimeSpecification(schedule): Returns: a TimeSpecification instance """ - parser = groc.CreateParser(schedule) parser.timespec() if parser.interval_mins: - return IntervalTimeSpecification(parser.interval_mins, parser.period_string) + return IntervalTimeSpecification(parser.interval_mins, + parser.period_string) else: return SpecificTimeSpecification(parser.ordinal_set, parser.weekday_set, - parser.month_set, None, parser.time_string) + parser.month_set, + None, + parser.time_string) class TimeSpecification(object): @@ -71,7 +86,7 @@ class TimeSpecification(object): """Returns the next n times that match the schedule, starting at time start. Arguments: - start: a datetime to start from. Matches will start from after this time + start: a datetime to start from. Matches will start from after this time. n: the number of matching times to return Returns: @@ -89,7 +104,7 @@ class TimeSpecification(object): Must be implemented in subclasses. Arguments: - start: a datetime to start with. Matches will start from this time + start: a datetime to start with. Matches will start from this time. Returns: a datetime object @@ -100,14 +115,14 @@ class TimeSpecification(object): class IntervalTimeSpecification(TimeSpecification): """A time specification for a given interval. - An Interval type spec runs at the given fixed interval. They have two + An Interval type spec runs at the given fixed interval. It has two attributes: - period - the type of interval, either "hours" or "minutes" + period - the type of interval, either "hours" or "minutes" interval - the number of units of type period. """ def __init__(self, interval, period): - super(IntervalTimeSpecification, self).__init__(self) + super(IntervalTimeSpecification, self).__init__() self.interval = interval self.period = period @@ -115,7 +130,7 @@ class IntervalTimeSpecification(TimeSpecification): """Returns the next match after time 't'. Arguments: - t: a datetime to start from. Matches will start from after this time + t: a datetime to start from. Matches will start from after this time. Returns: a datetime object @@ -129,47 +144,57 @@ class IntervalTimeSpecification(TimeSpecification): class SpecificTimeSpecification(TimeSpecification): """Specific time specification. - A Specific interval is more complex, but define a certain time to run, on - given days. They have the following attributes: + A Specific interval is more complex, but defines a certain time to run and + the days that it should run. It has the following attributes: time - the time of day to run, as "HH:MM" ordinals - first, second, third &c, as a set of integers in 1..5 - months - the months that this is valid, as a set of integers in 1..12 - weekdays - the days of the week to run this, 0=Sunday, 6=Saturday. - - The specific time interval can be quite complex. A schedule could look like + months - the months that this should run, as a set of integers in 1..12 + weekdays - the days of the week that this should run, as a set of integers, + 0=Sunday, 6=Saturday + timezone - the optional timezone as a string for this specification. + Defaults to UTC - valid entries are things like Australia/Victoria + or PST8PDT. + + A specific time schedule can be quite complex. A schedule could look like this: "1st,third sat,sun of jan,feb,mar 09:15" - In this case, ordinals would be [1,3], weekdays [0,6], months [1,2,3] and time - would be "09:15". + In this case, ordinals would be {1,3}, weekdays {0,6}, months {1,2,3} and + time would be "09:15". """ + timezone = None + def __init__(self, ordinals=None, weekdays=None, months=None, monthdays=None, - timestr='00:00'): + timestr='00:00', timezone=None): super(SpecificTimeSpecification, self).__init__(self) - if weekdays and monthdays: + if weekdays is not None and monthdays is not None: raise ValueError("can't supply both monthdays and weekdays") if ordinals is None: self.ordinals = set(range(1, 6)) else: - self.ordinals = ordinals + self.ordinals = set(ordinals) if weekdays is None: self.weekdays = set(range(7)) else: - self.weekdays = weekdays + self.weekdays = set(weekdays) if months is None: self.months = set(range(1, 13)) else: - self.months = months + self.months = set(months) if monthdays is None: self.monthdays = set() else: - self.monthdays = monthdays + self.monthdays = set(monthdays) hourstr, minutestr = timestr.split(':') self.time = datetime.time(int(hourstr), int(minutestr)) + if timezone: + if pytz is None: + raise ValueError("need pytz in order to specify a timezone") + self.timezone = pytz.timezone(timezone) def _MatchingDays(self, year, month): """Returns matching days for the given year and month. @@ -225,33 +250,55 @@ class SpecificTimeSpecification(TimeSpecification): """Returns the next time that matches the schedule after time start. Arguments: - start: a datetime to start with. Matches will start after this time + start: a UTC datetime to start from. Matches will start after this time Returns: a datetime object """ start_time = start + if self.timezone and pytz is not None: + if not start_time.tzinfo: + start_time = pytz.utc.localize(start_time) + start_time = start_time.astimezone(self.timezone) + start_time = start_time.replace(tzinfo=None) if self.months: - months = self._NextMonthGenerator(start.month, self.months) + months = self._NextMonthGenerator(start_time.month, self.months) while True: month, yearwraps = months.next() - candidate = start_time.replace(day=1, month=month, + candidate_month = start_time.replace(day=1, month=month, year=start_time.year + yearwraps) if self.monthdays: - _, last_day = calendar.monthrange(candidate.year, candidate.month) - day_matches = sorted([x for x in self.monthdays if x <= last_day]) + _, last_day = calendar.monthrange(candidate_month.year, + candidate_month.month) + day_matches = sorted(x for x in self.monthdays if x <= last_day) else: - day_matches = self._MatchingDays(candidate.year, month) + day_matches = self._MatchingDays(candidate_month.year, month) - if ((candidate.year, candidate.month) + if ((candidate_month.year, candidate_month.month) == (start_time.year, start_time.month)): day_matches = [x for x in day_matches if x >= start_time.day] - if day_matches and day_matches[0] == start_time.day: - if start_time.time() >= self.time: - day_matches.pop(0) - if not day_matches: - continue - out = candidate.replace(day=day_matches[0], hour=self.time.hour, - minute=self.time.minute, second=0, microsecond=0) - return out + while (day_matches and day_matches[0] == start_time.day + and start_time.time() >= self.time): + day_matches.pop(0) + while day_matches: + out = candidate_month.replace(day=day_matches[0], hour=self.time.hour, + + + minute=self.time.minute, second=0, + microsecond=0) + if self.timezone and pytz is not None: + try: + out = self.timezone.localize(out, is_dst=None) + except AmbiguousTimeError: + out = self.timezone.localize(out) + except NonExistentTimeError: + for _ in range(24): + out = out.replace(minute=1) + datetime.timedelta(minutes=60) + try: + out = self.timezone.localize(out) + except NonExistentTimeError: + continue + break + out = out.astimezone(pytz.utc) + return out google_appengine/google/appengine/cron/groctimespecification.py @@ -192,6 +192,74 @@ EXISTS_OPERATORS = set((datastore_pb.Query_Filter.EXISTS, )) +def Normalize(filters, orders): + """ Normalizes filter and order query components. + + The resulting components have the same effect as the given components if used + in a query. + + Returns: + (filter, orders) the reduced set of filters and orders + """ + + for f in filters: + if f.op() == datastore_pb.Query_Filter.IN and f.property_size() == 1: + f.set_op(datastore_pb.Query_Filter.EQUAL); + + eq_properties = set([f.property(0).name() for f in filters if f.op() == datastore_pb.Query_Filter.EQUAL]); + + remove_set = eq_properties.copy() + new_orders = [] + for o in orders: + if o.property() not in remove_set: + remove_set.add(o.property()) + new_orders.append(o) + orders = new_orders + + + if datastore_types._KEY_SPECIAL_PROPERTY in eq_properties: + orders = [] + + new_orders = [] + for o in orders: + if o.property() == datastore_types._KEY_SPECIAL_PROPERTY: + new_orders.append(o) + break + new_orders.append(o) + orders = new_orders + + return (filters, orders) + + +def RemoveNativelySupportedComponents(filters, orders): + """ Removes query components that are natively supported by the datastore. + + The resulting filters and orders should not be used in an actual query. + + Returns + (filters, orders) the reduced set of filters and orders + """ + (filters, orders) = Normalize(filters, orders) + + has_key_desc_order = False + if orders and orders[-1].property() == datastore_types._KEY_SPECIAL_PROPERTY: + if orders[-1].direction() == ASCENDING: + orders = orders[:-1] + else: + has_key_desc_order = True + + if not has_key_desc_order: + for f in filters: + if (f.op() in INEQUALITY_OPERATORS and + f.property(0).name() != datastore_types._KEY_SPECIAL_PROPERTY): + break + else: + filters = [f for f in filters + if f.property(0).name() != datastore_types._KEY_SPECIAL_PROPERTY] + + return (filters, orders) + + def CompositeIndexForQuery(query): """Return the composite index needed for a query. @@ -213,12 +281,18 @@ def CompositeIndexForQuery(query): can be at most one of these. - After that come all the (property, direction) pairs for the Order - entries, in the order given in the query. Exceptions: (a) if - there is a Filter entry with an inequality operator that matches - the first Order entry, the first order pair is omitted (or, - equivalently, in this case the inequality pair is omitted); (b) if - an Order entry corresponds to an equality filter, it is ignored - (since there will only ever be one value returned). + entries, in the order given in the query. Exceptions: + (a) if there is a Filter entry with an inequality operator that matches + the first Order entry, the first order pair is omitted (or, + equivalently, in this case the inequality pair is omitted). + (b) if an Order entry corresponds to an equality filter, it is ignored + (since there will only ever be one value returned). + (c) if there is an equality filter on __key__ all orders are dropped + (since there will be at most one result returned). + (d) if there is an order on __key__ all further orders are dropped (since + keys are unique). + (e) orders on __key__ ASCENDING are dropped (since this is supported + natively by the datastore). - Finally, if there are Filter entries whose operator is EXISTS, and whose property names are not already listed, they are added, with @@ -271,16 +345,18 @@ def CompositeIndexForQuery(query): nprops = len(filter.property_list()) assert nprops == 1, 'Filter has %s properties, expected 1' % nprops - if ancestor and not kind and not filters and not orders: + if not kind: required = False + (filters, orders) = RemoveNativelySupportedComponents(filters, orders) + eq_filters = [f for f in filters if f.op() in EQUALITY_OPERATORS] ineq_filters = [f for f in filters if f.op() in INEQUALITY_OPERATORS] exists_filters = [f for f in filters if f.op() in EXISTS_OPERATORS] assert (len(eq_filters) + len(ineq_filters) + len(exists_filters)) == len(filters), 'Not all filters used' - if (kind and eq_filters and not ineq_filters and not exists_filters and + if (kind and not ineq_filters and not exists_filters and not orders): names = set(f.property(0).name() for f in eq_filters) if not names.intersection(datastore_types._SPECIAL_PROPERTIES): @@ -292,16 +368,6 @@ def CompositeIndexForQuery(query): for filter in ineq_filters: assert filter.property(0).name() == ineq_property - new_orders = [] - for order in orders: - name = order.property() - for filter in eq_filters: - if filter.property(0).name() == name: - break - else: - new_orders.append(order) - orders = new_orders - props = [] for f in eq_filters: @@ -328,8 +394,7 @@ def CompositeIndexForQuery(query): else: props.append((prop_name, ASCENDING)) - if (kind and not ancestor and - (not props or (len(props) == 1 and props[0][1] == ASCENDING))): + if kind and not ancestor and len(props) <= 1: required = False if props: google_appengine/google/appengine/datastore/datastore_index.py @@ -25,10 +25,12 @@ __pychecker__ = """maxreturns=0 maxbranches=0 no-callinit from google.appengine.api.api_base_pb import Integer64Proto; from google.appengine.api.api_base_pb import StringProto; from google.appengine.api.api_base_pb import VoidProto; +from google.appengine.datastore.action_pb import Action from google.appengine.datastore.entity_pb import CompositeIndex from google.appengine.datastore.entity_pb import EntityProto from google.appengine.datastore.entity_pb import Index from google.appengine.datastore.entity_pb import Property +from google.appengine.datastore.entity_pb import Path from google.appengine.datastore.entity_pb import Reference class Transaction(ProtocolBuffer.ProtocolMessage): has_handle_ = 0 @@ -44,8 +46,9 @@ class Transaction(ProtocolBuffer.ProtocolMessage): self.handle_ = x def clear_handle(self): - self.has_handle_ = 0 - self.handle_ = 0 + if self.has_handle_: + self.has_handle_ = 0 + self.handle_ = 0 def has_handle(self): return self.has_handle_ @@ -94,18 +97,21 @@ class Transaction(ProtocolBuffer.ProtocolMessage): if self.has_handle_: res+=prefix+("handle: %s\n" % self.DebugFormatFixed64(self.handle_)) return res - khandle = 1 - _TEXT = ( - "ErrorCode", - "handle", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + khandle = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.DOUBLE, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "handle", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.DOUBLE, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -146,8 +152,9 @@ class Query_Filter(ProtocolBuffer.ProtocolMessage): self.op_ = x def clear_op(self): - self.has_op_ = 0 - self.op_ = 0 + if self.has_op_: + self.has_op_ = 0 + self.op_ = 0 def has_op(self): return self.has_op_ @@ -269,8 +276,9 @@ class Query_Order(ProtocolBuffer.ProtocolMessage): self.property_ = x def clear_property(self): - self.has_property_ = 0 - self.property_ = "" + if self.has_property_: + self.has_property_ = 0 + self.property_ = "" def has_property(self): return self.has_property_ @@ -281,8 +289,9 @@ class Query_Order(ProtocolBuffer.ProtocolMessage): self.direction_ = x def clear_direction(self): - self.has_direction_ = 0 - self.direction_ = 1 + if self.has_direction_: + self.has_direction_ = 0 + self.direction_ = 1 def has_direction(self): return self.has_direction_ @@ -370,12 +379,22 @@ class Query(ProtocolBuffer.ProtocolMessage): search_query_ = "" has_hint_ = 0 hint_ = 0 + has_count_ = 0 + count_ = 0 has_offset_ = 0 offset_ = 0 has_limit_ = 0 limit_ = 0 has_require_perfect_plan_ = 0 require_perfect_plan_ = 0 + has_keys_only_ = 0 + keys_only_ = 0 + has_transaction_ = 0 + transaction_ = None + has_distinct_ = 0 + distinct_ = 0 + has_compile_ = 0 + compile_ = 0 def __init__(self, contents=None): self.filter_ = [] @@ -391,8 +410,9 @@ class Query(ProtocolBuffer.ProtocolMessage): self.app_ = x def clear_app(self): - self.has_app_ = 0 - self.app_ = "" + if self.has_app_: + self.has_app_ = 0 + self.app_ = "" def has_app(self): return self.has_app_ @@ -403,8 +423,9 @@ class Query(ProtocolBuffer.ProtocolMessage): self.kind_ = x def clear_kind(self): - self.has_kind_ = 0 - self.kind_ = "" + if self.has_kind_: + self.has_kind_ = 0 + self.kind_ = "" def has_kind(self): return self.has_kind_ @@ -420,8 +441,9 @@ class Query(ProtocolBuffer.ProtocolMessage): def mutable_ancestor(self): self.has_ancestor_ = 1; return self.ancestor() def clear_ancestor(self): - self.has_ancestor_ = 0; - if self.ancestor_ is not None: self.ancestor_.Clear() + if self.has_ancestor_: + self.has_ancestor_ = 0; + if self.ancestor_ is not None: self.ancestor_.Clear() def has_ancestor(self): return self.has_ancestor_ @@ -448,8 +470,9 @@ class Query(ProtocolBuffer.ProtocolMessage): self.search_query_ = x def clear_search_query(self): - self.has_search_query_ = 0 - self.search_query_ = "" + if self.has_search_query_: + self.has_search_query_ = 0 + self.search_query_ = "" def has_search_query(self): return self.has_search_query_ @@ -476,11 +499,25 @@ class Query(ProtocolBuffer.ProtocolMessage): self.hint_ = x def clear_hint(self): - self.has_hint_ = 0 - self.hint_ = 0 + if self.has_hint_: + self.has_hint_ = 0 + self.hint_ = 0 def has_hint(self): return self.has_hint_ + def count(self): return self.count_ + + def set_count(self, x): + self.has_count_ = 1 + self.count_ = x + + def clear_count(self): + if self.has_count_: + self.has_count_ = 0 + self.count_ = 0 + + def has_count(self): return self.has_count_ + def offset(self): return self.offset_ def set_offset(self, x): @@ -488,8 +525,9 @@ class Query(ProtocolBuffer.ProtocolMessage): self.offset_ = x def clear_offset(self): - self.has_offset_ = 0 - self.offset_ = 0 + if self.has_offset_: + self.has_offset_ = 0 + self.offset_ = 0 def has_offset(self): return self.has_offset_ @@ -500,8 +538,9 @@ class Query(ProtocolBuffer.ProtocolMessage): self.limit_ = x def clear_limit(self): - self.has_limit_ = 0 - self.limit_ = 0 + if self.has_limit_: + self.has_limit_ = 0 + self.limit_ = 0 def has_limit(self): return self.has_limit_ @@ -528,11 +567,69 @@ class Query(ProtocolBuffer.ProtocolMessage): self.require_perfect_plan_ = x def clear_require_perfect_plan(self): - self.has_require_perfect_plan_ = 0 - self.require_perfect_plan_ = 0 + if self.has_require_perfect_plan_: + self.has_require_perfect_plan_ = 0 + self.require_perfect_plan_ = 0 def has_require_perfect_plan(self): return self.has_require_perfect_plan_ + def keys_only(self): return self.keys_only_ + + def set_keys_only(self, x): + self.has_keys_only_ = 1 + self.keys_only_ = x + + def clear_keys_only(self): + if self.has_keys_only_: + self.has_keys_only_ = 0 + self.keys_only_ = 0 + + def has_keys_only(self): return self.has_keys_only_ + + def transaction(self): + if self.transaction_ is None: + self.lazy_init_lock_.acquire() + try: + if self.transaction_ is None: self.transaction_ = Transaction() + finally: + self.lazy_init_lock_.release() + return self.transaction_ + + def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction() + + def clear_transaction(self): + if self.has_transaction_: + self.has_transaction_ = 0; + if self.transaction_ is not None: self.transaction_.Clear() + + def has_transaction(self): return self.has_transaction_ + + def distinct(self): return self.distinct_ + + def set_distinct(self, x): + self.has_distinct_ = 1 + self.distinct_ = x + + def clear_distinct(self): + if self.has_distinct_: + self.has_distinct_ = 0 + self.distinct_ = 0 + + def has_distinct(self): return self.has_distinct_ + + def compile(self): return self.compile_ + + def set_compile(self, x): + self.has_compile_ = 1 + self.compile_ = x + + def clear_compile(self): + if self.has_compile_: + self.has_compile_ = 0 + self.compile_ = 0 + + def has_compile(self): return self.has_compile_ + def MergeFrom(self, x): assert x is not self @@ -543,10 +640,15 @@ class Query(ProtocolBuffer.ProtocolMessage): if (x.has_search_query()): self.set_search_query(x.search_query()) for i in xrange(x.order_size()): self.add_order().CopyFrom(x.order(i)) if (x.has_hint()): self.set_hint(x.hint()) + if (x.has_count()): self.set_count(x.count()) if (x.has_offset()): self.set_offset(x.offset()) if (x.has_limit()): self.set_limit(x.limit()) for i in xrange(x.composite_index_size()): self.add_composite_index().CopyFrom(x.composite_index(i)) if (x.has_require_perfect_plan()): self.set_require_perfect_plan(x.require_perfect_plan()) + if (x.has_keys_only()): self.set_keys_only(x.keys_only()) + if (x.has_transaction()): self.mutable_transaction().MergeFrom(x.transaction()) + if (x.has_distinct()): self.set_distinct(x.distinct()) + if (x.has_compile()): self.set_compile(x.compile()) def Equals(self, x): if x is self: return 1 @@ -566,6 +668,8 @@ class Query(ProtocolBuffer.ProtocolMessage): if e1 != e2: return 0 if self.has_hint_ != x.has_hint_: return 0 if self.has_hint_ and self.hint_ != x.hint_: return 0 + if self.has_count_ != x.has_count_: return 0 + if self.has_count_ and self.count_ != x.count_: return 0 if self.has_offset_ != x.has_offset_: return 0 if self.has_offset_ and self.offset_ != x.offset_: return 0 if self.has_limit_ != x.has_limit_: return 0 @@ -575,6 +679,14 @@ class Query(ProtocolBuffer.ProtocolMessage): if e1 != e2: return 0 if self.has_require_perfect_plan_ != x.has_require_perfect_plan_: return 0 if self.has_require_perfect_plan_ and self.require_perfect_plan_ != x.require_perfect_plan_: return 0 + if self.has_keys_only_ != x.has_keys_only_: return 0 + if self.has_keys_only_ and self.keys_only_ != x.keys_only_: return 0 + if self.has_transaction_ != x.has_transaction_: return 0 + if self.has_transaction_ and self.transaction_ != x.transaction_: return 0 + if self.has_distinct_ != x.has_distinct_: return 0 + if self.has_distinct_ and self.distinct_ != x.distinct_: return 0 + if self.has_compile_ != x.has_compile_: return 0 + if self.has_compile_ and self.compile_ != x.compile_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -590,6 +702,7 @@ class Query(ProtocolBuffer.ProtocolMessage): if not p.IsInitialized(debug_strs): initialized=0 for p in self.composite_index_: if not p.IsInitialized(debug_strs): initialized=0 + if (self.has_transaction_ and not self.transaction_.IsInitialized(debug_strs)): initialized = 0 return initialized def ByteSize(self): @@ -603,11 +716,16 @@ class Query(ProtocolBuffer.ProtocolMessage): n += 2 * len(self.order_) for i in xrange(len(self.order_)): n += self.order_[i].ByteSize() if (self.has_hint_): n += 2 + self.lengthVarInt64(self.hint_) + if (self.has_count_): n += 2 + self.lengthVarInt64(self.count_) if (self.has_offset_): n += 1 + self.lengthVarInt64(self.offset_) if (self.has_limit_): n += 2 + self.lengthVarInt64(self.limit_) n += 2 * len(self.composite_index_) for i in xrange(len(self.composite_index_)): n += self.lengthString(self.composite_index_[i].ByteSize()) if (self.has_require_perfect_plan_): n += 3 + if (self.has_keys_only_): n += 3 + if (self.has_transaction_): n += 2 + self.lengthString(self.transaction_.ByteSize()) + if (self.has_distinct_): n += 3 + if (self.has_compile_): n += 3 return n + 1 def Clear(self): @@ -618,10 +736,15 @@ class Query(ProtocolBuffer.ProtocolMessage): self.clear_search_query() self.clear_order() self.clear_hint() + self.clear_count() self.clear_offset() self.clear_limit() self.clear_composite_index() self.clear_require_perfect_plan() + self.clear_keys_only() + self.clear_transaction() + self.clear_distinct() + self.clear_compile() def OutputUnchecked(self, out): out.putVarInt32(10) @@ -660,6 +783,22 @@ class Query(ProtocolBuffer.ProtocolMessage): if (self.has_require_perfect_plan_): out.putVarInt32(160) out.putBoolean(self.require_perfect_plan_) + if (self.has_keys_only_): + out.putVarInt32(168) + out.putBoolean(self.keys_only_) + if (self.has_transaction_): + out.putVarInt32(178) + out.putVarInt32(self.transaction_.ByteSize()) + self.transaction_.OutputUnchecked(out) + if (self.has_count_): + out.putVarInt32(184) + out.putVarInt32(self.count_) + if (self.has_distinct_): + out.putVarInt32(192) + out.putBoolean(self.distinct_) + if (self.has_compile_): + out.putVarInt32(200) + out.putBoolean(self.compile_) def TryMerge(self, d): while d.avail() > 0: @@ -703,6 +842,24 @@ class Query(ProtocolBuffer.ProtocolMessage): if tt == 160: self.set_require_perfect_plan(d.getBoolean()) continue + if tt == 168: + self.set_keys_only(d.getBoolean()) + continue + if tt == 178: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_transaction().TryMerge(tmp) + continue + if tt == 184: + self.set_count(d.getVarInt32()) + continue + if tt == 192: + self.set_distinct(d.getBoolean()) + continue + if tt == 200: + self.set_compile(d.getBoolean()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -733,6 +890,7 @@ class Query(ProtocolBuffer.ProtocolMessage): res+=prefix+"}\n" cnt+=1 if self.has_hint_: res+=prefix+("hint: %s\n" % self.DebugFormatInt32(self.hint_)) + if self.has_count_: res+=prefix+("count: %s\n" % self.DebugFormatInt32(self.count_)) if self.has_offset_: res+=prefix+("offset: %s\n" % self.DebugFormatInt32(self.offset_)) if self.has_limit_: res+=prefix+("limit: %s\n" % self.DebugFormatInt32(self.limit_)) cnt=0 @@ -744,8 +902,19 @@ class Query(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" cnt+=1 if self.has_require_perfect_plan_: res+=prefix+("require_perfect_plan: %s\n" % self.DebugFormatBool(self.require_perfect_plan_)) + if self.has_keys_only_: res+=prefix+("keys_only: %s\n" % self.DebugFormatBool(self.keys_only_)) + if self.has_transaction_: + res+=prefix+"transaction <\n" + res+=self.transaction_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + if self.has_distinct_: res+=prefix+("distinct: %s\n" % self.DebugFormatBool(self.distinct_)) + if self.has_compile_: res+=prefix+("compile: %s\n" % self.DebugFormatBool(self.compile_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kapp = 1 kkind = 3 kancestor = 17 @@ -757,78 +926,981 @@ class Query(ProtocolBuffer.ProtocolMessage): kOrderproperty = 10 kOrderdirection = 11 khint = 18 + kcount = 23 koffset = 12 klimit = 16 kcomposite_index = 19 krequire_perfect_plan = 20 + kkeys_only = 21 + ktransaction = 22 + kdistinct = 24 + kcompile = 25 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "app", + 3: "kind", + 4: "Filter", + 6: "op", + 8: "search_query", + 9: "Order", + 10: "property", + 11: "direction", + 12: "offset", + 14: "property", + 16: "limit", + 17: "ancestor", + 18: "hint", + 19: "composite_index", + 20: "require_perfect_plan", + 21: "keys_only", + 22: "transaction", + 23: "count", + 24: "distinct", + 25: "compile", + }, 25) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.STARTGROUP, + 6: ProtocolBuffer.Encoder.NUMERIC, + 8: ProtocolBuffer.Encoder.STRING, + 9: ProtocolBuffer.Encoder.STARTGROUP, + 10: ProtocolBuffer.Encoder.STRING, + 11: ProtocolBuffer.Encoder.NUMERIC, + 12: ProtocolBuffer.Encoder.NUMERIC, + 14: ProtocolBuffer.Encoder.STRING, + 16: ProtocolBuffer.Encoder.NUMERIC, + 17: ProtocolBuffer.Encoder.STRING, + 18: ProtocolBuffer.Encoder.NUMERIC, + 19: ProtocolBuffer.Encoder.STRING, + 20: ProtocolBuffer.Encoder.NUMERIC, + 21: ProtocolBuffer.Encoder.NUMERIC, + 22: ProtocolBuffer.Encoder.STRING, + 23: ProtocolBuffer.Encoder.NUMERIC, + 24: ProtocolBuffer.Encoder.NUMERIC, + 25: ProtocolBuffer.Encoder.NUMERIC, + }, 25, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CompiledQuery_PrimaryScan(ProtocolBuffer.ProtocolMessage): + has_index_name_ = 0 + index_name_ = "" + has_start_key_ = 0 + start_key_ = "" + has_start_inclusive_ = 0 + start_inclusive_ = 0 + has_end_key_ = 0 + end_key_ = "" + has_end_inclusive_ = 0 + end_inclusive_ = 0 + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def index_name(self): return self.index_name_ + + def set_index_name(self, x): + self.has_index_name_ = 1 + self.index_name_ = x + + def clear_index_name(self): + if self.has_index_name_: + self.has_index_name_ = 0 + self.index_name_ = "" + + def has_index_name(self): return self.has_index_name_ + + def start_key(self): return self.start_key_ + + def set_start_key(self, x): + self.has_start_key_ = 1 + self.start_key_ = x + + def clear_start_key(self): + if self.has_start_key_: + self.has_start_key_ = 0 + self.start_key_ = "" + + def has_start_key(self): return self.has_start_key_ + + def start_inclusive(self): return self.start_inclusive_ + + def set_start_inclusive(self, x): + self.has_start_inclusive_ = 1 + self.start_inclusive_ = x + + def clear_start_inclusive(self): + if self.has_start_inclusive_: + self.has_start_inclusive_ = 0 + self.start_inclusive_ = 0 + + def has_start_inclusive(self): return self.has_start_inclusive_ + + def end_key(self): return self.end_key_ + + def set_end_key(self, x): + self.has_end_key_ = 1 + self.end_key_ = x + + def clear_end_key(self): + if self.has_end_key_: + self.has_end_key_ = 0 + self.end_key_ = "" + + def has_end_key(self): return self.has_end_key_ + + def end_inclusive(self): return self.end_inclusive_ + + def set_end_inclusive(self, x): + self.has_end_inclusive_ = 1 + self.end_inclusive_ = x + + def clear_end_inclusive(self): + if self.has_end_inclusive_: + self.has_end_inclusive_ = 0 + self.end_inclusive_ = 0 + + def has_end_inclusive(self): return self.has_end_inclusive_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_index_name()): self.set_index_name(x.index_name()) + if (x.has_start_key()): self.set_start_key(x.start_key()) + if (x.has_start_inclusive()): self.set_start_inclusive(x.start_inclusive()) + if (x.has_end_key()): self.set_end_key(x.end_key()) + if (x.has_end_inclusive()): self.set_end_inclusive(x.end_inclusive()) + + def Equals(self, x): + if x is self: return 1 + if self.has_index_name_ != x.has_index_name_: return 0 + if self.has_index_name_ and self.index_name_ != x.index_name_: return 0 + if self.has_start_key_ != x.has_start_key_: return 0 + if self.has_start_key_ and self.start_key_ != x.start_key_: return 0 + if self.has_start_inclusive_ != x.has_start_inclusive_: return 0 + if self.has_start_inclusive_ and self.start_inclusive_ != x.start_inclusive_: return 0 + if self.has_end_key_ != x.has_end_key_: return 0 + if self.has_end_key_ and self.end_key_ != x.end_key_: return 0 + if self.has_end_inclusive_ != x.has_end_inclusive_: return 0 + if self.has_end_inclusive_ and self.end_inclusive_ != x.end_inclusive_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + return initialized + + def ByteSize(self): + n = 0 + if (self.has_index_name_): n += 1 + self.lengthString(len(self.index_name_)) + if (self.has_start_key_): n += 1 + self.lengthString(len(self.start_key_)) + if (self.has_start_inclusive_): n += 2 + if (self.has_end_key_): n += 1 + self.lengthString(len(self.end_key_)) + if (self.has_end_inclusive_): n += 2 + return n + 0 + + def Clear(self): + self.clear_index_name() + self.clear_start_key() + self.clear_start_inclusive() + self.clear_end_key() + self.clear_end_inclusive() + + def OutputUnchecked(self, out): + if (self.has_index_name_): + out.putVarInt32(18) + out.putPrefixedString(self.index_name_) + if (self.has_start_key_): + out.putVarInt32(26) + out.putPrefixedString(self.start_key_) + if (self.has_start_inclusive_): + out.putVarInt32(32) + out.putBoolean(self.start_inclusive_) + if (self.has_end_key_): + out.putVarInt32(42) + out.putPrefixedString(self.end_key_) + if (self.has_end_inclusive_): + out.putVarInt32(48) + out.putBoolean(self.end_inclusive_) + + def TryMerge(self, d): + while 1: + tt = d.getVarInt32() + if tt == 12: break + if tt == 18: + self.set_index_name(d.getPrefixedString()) + continue + if tt == 26: + self.set_start_key(d.getPrefixedString()) + continue + if tt == 32: + self.set_start_inclusive(d.getBoolean()) + continue + if tt == 42: + self.set_end_key(d.getPrefixedString()) + continue + if tt == 48: + self.set_end_inclusive(d.getBoolean()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_index_name_: res+=prefix+("index_name: %s\n" % self.DebugFormatString(self.index_name_)) + if self.has_start_key_: res+=prefix+("start_key: %s\n" % self.DebugFormatString(self.start_key_)) + if self.has_start_inclusive_: res+=prefix+("start_inclusive: %s\n" % self.DebugFormatBool(self.start_inclusive_)) + if self.has_end_key_: res+=prefix+("end_key: %s\n" % self.DebugFormatString(self.end_key_)) + if self.has_end_inclusive_: res+=prefix+("end_inclusive: %s\n" % self.DebugFormatBool(self.end_inclusive_)) + return res + +class CompiledQuery_MergeJoinScan(ProtocolBuffer.ProtocolMessage): + has_index_name_ = 0 + index_name_ = "" + + def __init__(self, contents=None): + self.prefix_value_ = [] + if contents is not None: self.MergeFromString(contents) + + def index_name(self): return self.index_name_ + + def set_index_name(self, x): + self.has_index_name_ = 1 + self.index_name_ = x + + def clear_index_name(self): + if self.has_index_name_: + self.has_index_name_ = 0 + self.index_name_ = "" + + def has_index_name(self): return self.has_index_name_ + + def prefix_value_size(self): return len(self.prefix_value_) + def prefix_value_list(self): return self.prefix_value_ + + def prefix_value(self, i): + return self.prefix_value_[i] + + def set_prefix_value(self, i, x): + self.prefix_value_[i] = x + + def add_prefix_value(self, x): + self.prefix_value_.append(x) + + def clear_prefix_value(self): + self.prefix_value_ = [] + + + def MergeFrom(self, x): + assert x is not self + if (x.has_index_name()): self.set_index_name(x.index_name()) + for i in xrange(x.prefix_value_size()): self.add_prefix_value(x.prefix_value(i)) + + def Equals(self, x): + if x is self: return 1 + if self.has_index_name_ != x.has_index_name_: return 0 + if self.has_index_name_ and self.index_name_ != x.index_name_: return 0 + if len(self.prefix_value_) != len(x.prefix_value_): return 0 + for e1, e2 in zip(self.prefix_value_, x.prefix_value_): + if e1 != e2: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_index_name_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: index_name not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(len(self.index_name_)) + n += 1 * len(self.prefix_value_) + for i in xrange(len(self.prefix_value_)): n += self.lengthString(len(self.prefix_value_[i])) + return n + 1 + + def Clear(self): + self.clear_index_name() + self.clear_prefix_value() + + def OutputUnchecked(self, out): + out.putVarInt32(66) + out.putPrefixedString(self.index_name_) + for i in xrange(len(self.prefix_value_)): + out.putVarInt32(74) + out.putPrefixedString(self.prefix_value_[i]) + + def TryMerge(self, d): + while 1: + tt = d.getVarInt32() + if tt == 60: break + if tt == 66: + self.set_index_name(d.getPrefixedString()) + continue + if tt == 74: + self.add_prefix_value(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_index_name_: res+=prefix+("index_name: %s\n" % self.DebugFormatString(self.index_name_)) + cnt=0 + for e in self.prefix_value_: + elm="" + if printElemNumber: elm="(%d)" % cnt + res+=prefix+("prefix_value%s: %s\n" % (elm, self.DebugFormatString(e))) + cnt+=1 + return res + +class CompiledQuery_EntityFilter(ProtocolBuffer.ProtocolMessage): + has_distinct_ = 0 + distinct_ = 0 + has_offset_ = 0 + offset_ = 0 + has_limit_ = 0 + limit_ = 0 + has_kind_ = 0 + kind_ = "" + has_ancestor_ = 0 + ancestor_ = None - _TEXT = ( - "ErrorCode", - "app", - None, - "kind", - "Filter", - None, - "op", - None, - "search_query", - "Order", - "property", - "direction", - "offset", - None, - "property", - None, - "limit", - "ancestor", - "hint", - "composite_index", - "require_perfect_plan", - ) + def __init__(self, contents=None): + self.lazy_init_lock_ = thread.allocate_lock() + if contents is not None: self.MergeFromString(contents) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + def distinct(self): return self.distinct_ - ProtocolBuffer.Encoder.MAX_TYPE, + def set_distinct(self, x): + self.has_distinct_ = 1 + self.distinct_ = x - ProtocolBuffer.Encoder.STRING, + def clear_distinct(self): + if self.has_distinct_: + self.has_distinct_ = 0 + self.distinct_ = 0 - ProtocolBuffer.Encoder.STARTGROUP, + def has_distinct(self): return self.has_distinct_ - ProtocolBuffer.Encoder.MAX_TYPE, + def offset(self): return self.offset_ - ProtocolBuffer.Encoder.NUMERIC, + def set_offset(self, x): + self.has_offset_ = 1 + self.offset_ = x - ProtocolBuffer.Encoder.MAX_TYPE, + def clear_offset(self): + if self.has_offset_: + self.has_offset_ = 0 + self.offset_ = 0 - ProtocolBuffer.Encoder.STRING, + def has_offset(self): return self.has_offset_ - ProtocolBuffer.Encoder.STARTGROUP, + def limit(self): return self.limit_ - ProtocolBuffer.Encoder.STRING, + def set_limit(self, x): + self.has_limit_ = 1 + self.limit_ = x - ProtocolBuffer.Encoder.NUMERIC, + def clear_limit(self): + if self.has_limit_: + self.has_limit_ = 0 + self.limit_ = 0 - ProtocolBuffer.Encoder.NUMERIC, + def has_limit(self): return self.has_limit_ - ProtocolBuffer.Encoder.MAX_TYPE, + def kind(self): return self.kind_ - ProtocolBuffer.Encoder.STRING, + def set_kind(self, x): + self.has_kind_ = 1 + self.kind_ = x - ProtocolBuffer.Encoder.MAX_TYPE, + def clear_kind(self): + if self.has_kind_: + self.has_kind_ = 0 + self.kind_ = "" - ProtocolBuffer.Encoder.NUMERIC, + def has_kind(self): return self.has_kind_ - ProtocolBuffer.Encoder.STRING, + def ancestor(self): + if self.ancestor_ is None: + self.lazy_init_lock_.acquire() + try: + if self.ancestor_ is None: self.ancestor_ = Reference() + finally: + self.lazy_init_lock_.release() + return self.ancestor_ - ProtocolBuffer.Encoder.NUMERIC, + def mutable_ancestor(self): self.has_ancestor_ = 1; return self.ancestor() - ProtocolBuffer.Encoder.STRING, + def clear_ancestor(self): + if self.has_ancestor_: + self.has_ancestor_ = 0; + if self.ancestor_ is not None: self.ancestor_.Clear() + + def has_ancestor(self): return self.has_ancestor_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_distinct()): self.set_distinct(x.distinct()) + if (x.has_offset()): self.set_offset(x.offset()) + if (x.has_limit()): self.set_limit(x.limit()) + if (x.has_kind()): self.set_kind(x.kind()) + if (x.has_ancestor()): self.mutable_ancestor().MergeFrom(x.ancestor()) + + def Equals(self, x): + if x is self: return 1 + if self.has_distinct_ != x.has_distinct_: return 0 + if self.has_distinct_ and self.distinct_ != x.distinct_: return 0 + if self.has_offset_ != x.has_offset_: return 0 + if self.has_offset_ and self.offset_ != x.offset_: return 0 + if self.has_limit_ != x.has_limit_: return 0 + if self.has_limit_ and self.limit_ != x.limit_: return 0 + if self.has_kind_ != x.has_kind_: return 0 + if self.has_kind_ and self.kind_ != x.kind_: return 0 + if self.has_ancestor_ != x.has_ancestor_: return 0 + if self.has_ancestor_ and self.ancestor_ != x.ancestor_: return 0 + return 1 - ProtocolBuffer.Encoder.NUMERIC, + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (self.has_ancestor_ and not self.ancestor_.IsInitialized(debug_strs)): initialized = 0 + return initialized - ) + def ByteSize(self): + n = 0 + if (self.has_distinct_): n += 2 + if (self.has_offset_): n += 1 + self.lengthVarInt64(self.offset_) + if (self.has_limit_): n += 2 + self.lengthVarInt64(self.limit_) + if (self.has_kind_): n += 2 + self.lengthString(len(self.kind_)) + if (self.has_ancestor_): n += 2 + self.lengthString(self.ancestor_.ByteSize()) + return n + 0 + + def Clear(self): + self.clear_distinct() + self.clear_offset() + self.clear_limit() + self.clear_kind() + self.clear_ancestor() + + def OutputUnchecked(self, out): + if (self.has_distinct_): + out.putVarInt32(112) + out.putBoolean(self.distinct_) + if (self.has_offset_): + out.putVarInt32(120) + out.putVarInt32(self.offset_) + if (self.has_limit_): + out.putVarInt32(128) + out.putVarInt32(self.limit_) + if (self.has_kind_): + out.putVarInt32(138) + out.putPrefixedString(self.kind_) + if (self.has_ancestor_): + out.putVarInt32(146) + out.putVarInt32(self.ancestor_.ByteSize()) + self.ancestor_.OutputUnchecked(out) + + def TryMerge(self, d): + while 1: + tt = d.getVarInt32() + if tt == 108: break + if tt == 112: + self.set_distinct(d.getBoolean()) + continue + if tt == 120: + self.set_offset(d.getVarInt32()) + continue + if tt == 128: + self.set_limit(d.getVarInt32()) + continue + if tt == 138: + self.set_kind(d.getPrefixedString()) + continue + if tt == 146: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_ancestor().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_distinct_: res+=prefix+("distinct: %s\n" % self.DebugFormatBool(self.distinct_)) + if self.has_offset_: res+=prefix+("offset: %s\n" % self.DebugFormatInt32(self.offset_)) + if self.has_limit_: res+=prefix+("limit: %s\n" % self.DebugFormatInt32(self.limit_)) + if self.has_kind_: res+=prefix+("kind: %s\n" % self.DebugFormatString(self.kind_)) + if self.has_ancestor_: + res+=prefix+"ancestor <\n" + res+=self.ancestor_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + +class CompiledQuery(ProtocolBuffer.ProtocolMessage): + has_primaryscan_ = 0 + has_offset_ = 0 + offset_ = 0 + has_limit_ = 0 + limit_ = 0 + has_keys_only_ = 0 + keys_only_ = 0 + has_entityfilter_ = 0 + entityfilter_ = None + + def __init__(self, contents=None): + self.primaryscan_ = CompiledQuery_PrimaryScan() + self.mergejoinscan_ = [] + self.lazy_init_lock_ = thread.allocate_lock() + if contents is not None: self.MergeFromString(contents) + + def primaryscan(self): return self.primaryscan_ + + def mutable_primaryscan(self): self.has_primaryscan_ = 1; return self.primaryscan_ + + def clear_primaryscan(self):self.has_primaryscan_ = 0; self.primaryscan_.Clear() + + def has_primaryscan(self): return self.has_primaryscan_ + + def mergejoinscan_size(self): return len(self.mergejoinscan_) + def mergejoinscan_list(self): return self.mergejoinscan_ + + def mergejoinscan(self, i): + return self.mergejoinscan_[i] + + def mutable_mergejoinscan(self, i): + return self.mergejoinscan_[i] + + def add_mergejoinscan(self): + x = CompiledQuery_MergeJoinScan() + self.mergejoinscan_.append(x) + return x + + def clear_mergejoinscan(self): + self.mergejoinscan_ = [] + def offset(self): return self.offset_ + + def set_offset(self, x): + self.has_offset_ = 1 + self.offset_ = x + + def clear_offset(self): + if self.has_offset_: + self.has_offset_ = 0 + self.offset_ = 0 + + def has_offset(self): return self.has_offset_ + + def limit(self): return self.limit_ + + def set_limit(self, x): + self.has_limit_ = 1 + self.limit_ = x + + def clear_limit(self): + if self.has_limit_: + self.has_limit_ = 0 + self.limit_ = 0 + + def has_limit(self): return self.has_limit_ + + def keys_only(self): return self.keys_only_ + + def set_keys_only(self, x): + self.has_keys_only_ = 1 + self.keys_only_ = x + + def clear_keys_only(self): + if self.has_keys_only_: + self.has_keys_only_ = 0 + self.keys_only_ = 0 + + def has_keys_only(self): return self.has_keys_only_ + + def entityfilter(self): + if self.entityfilter_ is None: + self.lazy_init_lock_.acquire() + try: + if self.entityfilter_ is None: self.entityfilter_ = CompiledQuery_EntityFilter() + finally: + self.lazy_init_lock_.release() + return self.entityfilter_ + + def mutable_entityfilter(self): self.has_entityfilter_ = 1; return self.entityfilter() + + def clear_entityfilter(self): + if self.has_entityfilter_: + self.has_entityfilter_ = 0; + if self.entityfilter_ is not None: self.entityfilter_.Clear() + + def has_entityfilter(self): return self.has_entityfilter_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_primaryscan()): self.mutable_primaryscan().MergeFrom(x.primaryscan()) + for i in xrange(x.mergejoinscan_size()): self.add_mergejoinscan().CopyFrom(x.mergejoinscan(i)) + if (x.has_offset()): self.set_offset(x.offset()) + if (x.has_limit()): self.set_limit(x.limit()) + if (x.has_keys_only()): self.set_keys_only(x.keys_only()) + if (x.has_entityfilter()): self.mutable_entityfilter().MergeFrom(x.entityfilter()) + + def Equals(self, x): + if x is self: return 1 + if self.has_primaryscan_ != x.has_primaryscan_: return 0 + if self.has_primaryscan_ and self.primaryscan_ != x.primaryscan_: return 0 + if len(self.mergejoinscan_) != len(x.mergejoinscan_): return 0 + for e1, e2 in zip(self.mergejoinscan_, x.mergejoinscan_): + if e1 != e2: return 0 + if self.has_offset_ != x.has_offset_: return 0 + if self.has_offset_ and self.offset_ != x.offset_: return 0 + if self.has_limit_ != x.has_limit_: return 0 + if self.has_limit_ and self.limit_ != x.limit_: return 0 + if self.has_keys_only_ != x.has_keys_only_: return 0 + if self.has_keys_only_ and self.keys_only_ != x.keys_only_: return 0 + if self.has_entityfilter_ != x.has_entityfilter_: return 0 + if self.has_entityfilter_ and self.entityfilter_ != x.entityfilter_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_primaryscan_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: primaryscan not set.') + elif not self.primaryscan_.IsInitialized(debug_strs): initialized = 0 + for p in self.mergejoinscan_: + if not p.IsInitialized(debug_strs): initialized=0 + if (not self.has_keys_only_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: keys_only not set.') + if (self.has_entityfilter_ and not self.entityfilter_.IsInitialized(debug_strs)): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += self.primaryscan_.ByteSize() + n += 2 * len(self.mergejoinscan_) + for i in xrange(len(self.mergejoinscan_)): n += self.mergejoinscan_[i].ByteSize() + if (self.has_offset_): n += 1 + self.lengthVarInt64(self.offset_) + if (self.has_limit_): n += 1 + self.lengthVarInt64(self.limit_) + if (self.has_entityfilter_): n += 2 + self.entityfilter_.ByteSize() + return n + 4 + + def Clear(self): + self.clear_primaryscan() + self.clear_mergejoinscan() + self.clear_offset() + self.clear_limit() + self.clear_keys_only() + self.clear_entityfilter() + + def OutputUnchecked(self, out): + out.putVarInt32(11) + self.primaryscan_.OutputUnchecked(out) + out.putVarInt32(12) + for i in xrange(len(self.mergejoinscan_)): + out.putVarInt32(59) + self.mergejoinscan_[i].OutputUnchecked(out) + out.putVarInt32(60) + if (self.has_offset_): + out.putVarInt32(80) + out.putVarInt32(self.offset_) + if (self.has_limit_): + out.putVarInt32(88) + out.putVarInt32(self.limit_) + out.putVarInt32(96) + out.putBoolean(self.keys_only_) + if (self.has_entityfilter_): + out.putVarInt32(107) + self.entityfilter_.OutputUnchecked(out) + out.putVarInt32(108) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 11: + self.mutable_primaryscan().TryMerge(d) + continue + if tt == 59: + self.add_mergejoinscan().TryMerge(d) + continue + if tt == 80: + self.set_offset(d.getVarInt32()) + continue + if tt == 88: + self.set_limit(d.getVarInt32()) + continue + if tt == 96: + self.set_keys_only(d.getBoolean()) + continue + if tt == 107: + self.mutable_entityfilter().TryMerge(d) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_primaryscan_: + res+=prefix+"PrimaryScan {\n" + res+=self.primaryscan_.__str__(prefix + " ", printElemNumber) + res+=prefix+"}\n" + cnt=0 + for e in self.mergejoinscan_: + elm="" + if printElemNumber: elm="(%d)" % cnt + res+=prefix+("MergeJoinScan%s {\n" % elm) + res+=e.__str__(prefix + " ", printElemNumber) + res+=prefix+"}\n" + cnt+=1 + if self.has_offset_: res+=prefix+("offset: %s\n" % self.DebugFormatInt32(self.offset_)) + if self.has_limit_: res+=prefix+("limit: %s\n" % self.DebugFormatInt32(self.limit_)) + if self.has_keys_only_: res+=prefix+("keys_only: %s\n" % self.DebugFormatBool(self.keys_only_)) + if self.has_entityfilter_: + res+=prefix+"EntityFilter {\n" + res+=self.entityfilter_.__str__(prefix + " ", printElemNumber) + res+=prefix+"}\n" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kPrimaryScanGroup = 1 + kPrimaryScanindex_name = 2 + kPrimaryScanstart_key = 3 + kPrimaryScanstart_inclusive = 4 + kPrimaryScanend_key = 5 + kPrimaryScanend_inclusive = 6 + kMergeJoinScanGroup = 7 + kMergeJoinScanindex_name = 8 + kMergeJoinScanprefix_value = 9 + koffset = 10 + klimit = 11 + kkeys_only = 12 + kEntityFilterGroup = 13 + kEntityFilterdistinct = 14 + kEntityFilteroffset = 15 + kEntityFilterlimit = 16 + kEntityFilterkind = 17 + kEntityFilterancestor = 18 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "PrimaryScan", + 2: "index_name", + 3: "start_key", + 4: "start_inclusive", + 5: "end_key", + 6: "end_inclusive", + 7: "MergeJoinScan", + 8: "index_name", + 9: "prefix_value", + 10: "offset", + 11: "limit", + 12: "keys_only", + 13: "EntityFilter", + 14: "distinct", + 15: "offset", + 16: "limit", + 17: "kind", + 18: "ancestor", + }, 18) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STARTGROUP, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.STRING, + 6: ProtocolBuffer.Encoder.NUMERIC, + 7: ProtocolBuffer.Encoder.STARTGROUP, + 8: ProtocolBuffer.Encoder.STRING, + 9: ProtocolBuffer.Encoder.STRING, + 10: ProtocolBuffer.Encoder.NUMERIC, + 11: ProtocolBuffer.Encoder.NUMERIC, + 12: ProtocolBuffer.Encoder.NUMERIC, + 13: ProtocolBuffer.Encoder.STARTGROUP, + 14: ProtocolBuffer.Encoder.NUMERIC, + 15: ProtocolBuffer.Encoder.NUMERIC, + 16: ProtocolBuffer.Encoder.NUMERIC, + 17: ProtocolBuffer.Encoder.STRING, + 18: ProtocolBuffer.Encoder.STRING, + }, 18, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class RunCompiledQueryRequest(ProtocolBuffer.ProtocolMessage): + has_compiled_query_ = 0 + has_original_query_ = 0 + original_query_ = None + has_count_ = 0 + count_ = 0 + + def __init__(self, contents=None): + self.compiled_query_ = CompiledQuery() + self.lazy_init_lock_ = thread.allocate_lock() + if contents is not None: self.MergeFromString(contents) + + def compiled_query(self): return self.compiled_query_ + + def mutable_compiled_query(self): self.has_compiled_query_ = 1; return self.compiled_query_ + + def clear_compiled_query(self):self.has_compiled_query_ = 0; self.compiled_query_.Clear() + + def has_compiled_query(self): return self.has_compiled_query_ + + def original_query(self): + if self.original_query_ is None: + self.lazy_init_lock_.acquire() + try: + if self.original_query_ is None: self.original_query_ = Query() + finally: + self.lazy_init_lock_.release() + return self.original_query_ + + def mutable_original_query(self): self.has_original_query_ = 1; return self.original_query() + + def clear_original_query(self): + if self.has_original_query_: + self.has_original_query_ = 0; + if self.original_query_ is not None: self.original_query_.Clear() + + def has_original_query(self): return self.has_original_query_ + + def count(self): return self.count_ + + def set_count(self, x): + self.has_count_ = 1 + self.count_ = x + + def clear_count(self): + if self.has_count_: + self.has_count_ = 0 + self.count_ = 0 + + def has_count(self): return self.has_count_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_compiled_query()): self.mutable_compiled_query().MergeFrom(x.compiled_query()) + if (x.has_original_query()): self.mutable_original_query().MergeFrom(x.original_query()) + if (x.has_count()): self.set_count(x.count()) + + def Equals(self, x): + if x is self: return 1 + if self.has_compiled_query_ != x.has_compiled_query_: return 0 + if self.has_compiled_query_ and self.compiled_query_ != x.compiled_query_: return 0 + if self.has_original_query_ != x.has_original_query_: return 0 + if self.has_original_query_ and self.original_query_ != x.original_query_: return 0 + if self.has_count_ != x.has_count_: return 0 + if self.has_count_ and self.count_ != x.count_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_compiled_query_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: compiled_query not set.') + elif not self.compiled_query_.IsInitialized(debug_strs): initialized = 0 + if (self.has_original_query_ and not self.original_query_.IsInitialized(debug_strs)): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(self.compiled_query_.ByteSize()) + if (self.has_original_query_): n += 1 + self.lengthString(self.original_query_.ByteSize()) + if (self.has_count_): n += 1 + self.lengthVarInt64(self.count_) + return n + 1 + + def Clear(self): + self.clear_compiled_query() + self.clear_original_query() + self.clear_count() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putVarInt32(self.compiled_query_.ByteSize()) + self.compiled_query_.OutputUnchecked(out) + if (self.has_original_query_): + out.putVarInt32(18) + out.putVarInt32(self.original_query_.ByteSize()) + self.original_query_.OutputUnchecked(out) + if (self.has_count_): + out.putVarInt32(24) + out.putVarInt32(self.count_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_compiled_query().TryMerge(tmp) + continue + if tt == 18: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_original_query().TryMerge(tmp) + continue + if tt == 24: + self.set_count(d.getVarInt32()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_compiled_query_: + res+=prefix+"compiled_query <\n" + res+=self.compiled_query_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + if self.has_original_query_: + res+=prefix+"original_query <\n" + res+=self.original_query_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + if self.has_count_: res+=prefix+("count: %s\n" % self.DebugFormatInt32(self.count_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kcompiled_query = 1 + koriginal_query = 2 + kcount = 3 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "compiled_query", + 2: "original_query", + 3: "count", + }, 3) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.NUMERIC, + }, 3, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -851,8 +1923,9 @@ class QueryExplanation(ProtocolBuffer.ProtocolMessage): self.native_ancestor_ = x def clear_native_ancestor(self): - self.has_native_ancestor_ = 0 - self.native_ancestor_ = 0 + if self.has_native_ancestor_: + self.has_native_ancestor_ = 0 + self.native_ancestor_ = 0 def has_native_ancestor(self): return self.has_native_ancestor_ @@ -879,8 +1952,9 @@ class QueryExplanation(ProtocolBuffer.ProtocolMessage): self.native_offset_ = x def clear_native_offset(self): - self.has_native_offset_ = 0 - self.native_offset_ = 0 + if self.has_native_offset_: + self.has_native_offset_ = 0 + self.native_offset_ = 0 def has_native_offset(self): return self.has_native_offset_ @@ -891,8 +1965,9 @@ class QueryExplanation(ProtocolBuffer.ProtocolMessage): self.native_limit_ = x def clear_native_limit(self): - self.has_native_limit_ = 0 - self.native_limit_ = 0 + if self.has_native_limit_: + self.has_native_limit_ = 0 + self.native_limit_ = 0 def has_native_limit(self): return self.has_native_limit_ @@ -990,30 +2065,30 @@ class QueryExplanation(ProtocolBuffer.ProtocolMessage): if self.has_native_limit_: res+=prefix+("native_limit: %s\n" % self.DebugFormatInt32(self.native_limit_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + knative_ancestor = 1 knative_index = 2 knative_offset = 3 knative_limit = 4 - _TEXT = ( - "ErrorCode", - "native_ancestor", - "native_index", - "native_offset", - "native_limit", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "native_ancestor", + 2: "native_index", + 3: "native_offset", + 4: "native_limit", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.NUMERIC, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1031,8 +2106,9 @@ class Cursor(ProtocolBuffer.ProtocolMessage): self.cursor_ = x def clear_cursor(self): - self.has_cursor_ = 0 - self.cursor_ = 0 + if self.has_cursor_: + self.has_cursor_ = 0 + self.cursor_ = 0 def has_cursor(self): return self.has_cursor_ @@ -1081,18 +2157,21 @@ class Cursor(ProtocolBuffer.ProtocolMessage): if self.has_cursor_: res+=prefix+("cursor: %s\n" % self.DebugFormatFixed64(self.cursor_)) return res - kcursor = 1 - _TEXT = ( - "ErrorCode", - "cursor", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kcursor = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.DOUBLE, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "cursor", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.DOUBLE, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1103,6 +2182,7 @@ class Error(ProtocolBuffer.ProtocolMessage): INTERNAL_ERROR = 3 NEED_INDEX = 4 TIMEOUT = 5 + PERMISSION_DENIED = 6 _ErrorCode_NAMES = { 1: "BAD_REQUEST", @@ -1110,6 +2190,7 @@ class Error(ProtocolBuffer.ProtocolMessage): 3: "INTERNAL_ERROR", 4: "NEED_INDEX", 5: "TIMEOUT", + 6: "PERMISSION_DENIED", } def ErrorCode_Name(cls, x): return cls._ErrorCode_NAMES.get(x, "") @@ -1154,19 +2235,29 @@ class Error(ProtocolBuffer.ProtocolMessage): return res - _TEXT = ( - "ErrorCode", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" class Cost(ProtocolBuffer.ProtocolMessage): has_index_writes_ = 0 index_writes_ = 0 + has_index_write_bytes_ = 0 + index_write_bytes_ = 0 + has_entity_writes_ = 0 + entity_writes_ = 0 + has_entity_write_bytes_ = 0 + entity_write_bytes_ = 0 def __init__(self, contents=None): if contents is not None: self.MergeFromString(contents) @@ -1178,20 +2269,69 @@ class Cost(ProtocolBuffer.ProtocolMessage): self.index_writes_ = x def clear_index_writes(self): - self.has_index_writes_ = 0 - self.index_writes_ = 0 + if self.has_index_writes_: + self.has_index_writes_ = 0 + self.index_writes_ = 0 def has_index_writes(self): return self.has_index_writes_ + def index_write_bytes(self): return self.index_write_bytes_ + + def set_index_write_bytes(self, x): + self.has_index_write_bytes_ = 1 + self.index_write_bytes_ = x + + def clear_index_write_bytes(self): + if self.has_index_write_bytes_: + self.has_index_write_bytes_ = 0 + self.index_write_bytes_ = 0 + + def has_index_write_bytes(self): return self.has_index_write_bytes_ + + def entity_writes(self): return self.entity_writes_ + + def set_entity_writes(self, x): + self.has_entity_writes_ = 1 + self.entity_writes_ = x + + def clear_entity_writes(self): + if self.has_entity_writes_: + self.has_entity_writes_ = 0 + self.entity_writes_ = 0 + + def has_entity_writes(self): return self.has_entity_writes_ + + def entity_write_bytes(self): return self.entity_write_bytes_ + + def set_entity_write_bytes(self, x): + self.has_entity_write_bytes_ = 1 + self.entity_write_bytes_ = x + + def clear_entity_write_bytes(self): + if self.has_entity_write_bytes_: + self.has_entity_write_bytes_ = 0 + self.entity_write_bytes_ = 0 + + def has_entity_write_bytes(self): return self.has_entity_write_bytes_ + def MergeFrom(self, x): assert x is not self if (x.has_index_writes()): self.set_index_writes(x.index_writes()) + if (x.has_index_write_bytes()): self.set_index_write_bytes(x.index_write_bytes()) + if (x.has_entity_writes()): self.set_entity_writes(x.entity_writes()) + if (x.has_entity_write_bytes()): self.set_entity_write_bytes(x.entity_write_bytes()) def Equals(self, x): if x is self: return 1 if self.has_index_writes_ != x.has_index_writes_: return 0 if self.has_index_writes_ and self.index_writes_ != x.index_writes_: return 0 + if self.has_index_write_bytes_ != x.has_index_write_bytes_: return 0 + if self.has_index_write_bytes_ and self.index_write_bytes_ != x.index_write_bytes_: return 0 + if self.has_entity_writes_ != x.has_entity_writes_: return 0 + if self.has_entity_writes_ and self.entity_writes_ != x.entity_writes_: return 0 + if self.has_entity_write_bytes_ != x.has_entity_write_bytes_: return 0 + if self.has_entity_write_bytes_ and self.entity_write_bytes_ != x.entity_write_bytes_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -1201,15 +2341,30 @@ class Cost(ProtocolBuffer.ProtocolMessage): def ByteSize(self): n = 0 if (self.has_index_writes_): n += 1 + self.lengthVarInt64(self.index_writes_) + if (self.has_index_write_bytes_): n += 1 + self.lengthVarInt64(self.index_write_bytes_) + if (self.has_entity_writes_): n += 1 + self.lengthVarInt64(self.entity_writes_) + if (self.has_entity_write_bytes_): n += 1 + self.lengthVarInt64(self.entity_write_bytes_) return n + 0 def Clear(self): self.clear_index_writes() + self.clear_index_write_bytes() + self.clear_entity_writes() + self.clear_entity_write_bytes() def OutputUnchecked(self, out): if (self.has_index_writes_): out.putVarInt32(8) out.putVarInt32(self.index_writes_) + if (self.has_index_write_bytes_): + out.putVarInt32(16) + out.putVarInt32(self.index_write_bytes_) + if (self.has_entity_writes_): + out.putVarInt32(24) + out.putVarInt32(self.entity_writes_) + if (self.has_entity_write_bytes_): + out.putVarInt32(32) + out.putVarInt32(self.entity_write_bytes_) def TryMerge(self, d): while d.avail() > 0: @@ -1217,6 +2372,15 @@ class Cost(ProtocolBuffer.ProtocolMessage): if tt == 8: self.set_index_writes(d.getVarInt32()) continue + if tt == 16: + self.set_index_write_bytes(d.getVarInt32()) + continue + if tt == 24: + self.set_entity_writes(d.getVarInt32()) + continue + if tt == 32: + self.set_entity_write_bytes(d.getVarInt32()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -1224,20 +2388,35 @@ class Cost(ProtocolBuffer.ProtocolMessage): def __str__(self, prefix="", printElemNumber=0): res="" if self.has_index_writes_: res+=prefix+("index_writes: %s\n" % self.DebugFormatInt32(self.index_writes_)) + if self.has_index_write_bytes_: res+=prefix+("index_write_bytes: %s\n" % self.DebugFormatInt32(self.index_write_bytes_)) + if self.has_entity_writes_: res+=prefix+("entity_writes: %s\n" % self.DebugFormatInt32(self.entity_writes_)) + if self.has_entity_write_bytes_: res+=prefix+("entity_write_bytes: %s\n" % self.DebugFormatInt32(self.entity_write_bytes_)) return res - kindex_writes = 1 - _TEXT = ( - "ErrorCode", - "index_writes", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ) + kindex_writes = 1 + kindex_write_bytes = 2 + kentity_writes = 3 + kentity_write_bytes = 4 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "index_writes", + 2: "index_write_bytes", + 3: "entity_writes", + 4: "entity_write_bytes", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.NUMERIC, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1278,8 +2457,9 @@ class GetRequest(ProtocolBuffer.ProtocolMessage): def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction() def clear_transaction(self): - self.has_transaction_ = 0; - if self.transaction_ is not None: self.transaction_.Clear() + if self.has_transaction_: + self.has_transaction_ = 0; + if self.transaction_ is not None: self.transaction_.Clear() def has_transaction(self): return self.has_transaction_ @@ -1361,22 +2541,24 @@ class GetRequest(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res - kkey = 1 - ktransaction = 2 - _TEXT = ( - "ErrorCode", - "key", - "transaction", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + kkey = 1 + ktransaction = 2 - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "key", + 2: "transaction", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1400,8 +2582,9 @@ class GetResponse_Entity(ProtocolBuffer.ProtocolMessage): def mutable_entity(self): self.has_entity_ = 1; return self.entity() def clear_entity(self): - self.has_entity_ = 0; - if self.entity_ is not None: self.entity_.Clear() + if self.has_entity_: + self.has_entity_ = 0; + if self.entity_ is not None: self.entity_.Clear() def has_entity(self): return self.has_entity_ @@ -1534,28 +2717,32 @@ class GetResponse(ProtocolBuffer.ProtocolMessage): cnt+=1 return res - kEntityGroup = 1 - kEntityentity = 2 - _TEXT = ( - "ErrorCode", - "Entity", - "entity", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STARTGROUP, + kEntityGroup = 1 + kEntityentity = 2 - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Entity", + 2: "entity", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STARTGROUP, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" class PutRequest(ProtocolBuffer.ProtocolMessage): has_transaction_ = 0 transaction_ = None + has_trusted_ = 0 + trusted_ = 0 def __init__(self, contents=None): self.entity_ = [] @@ -1591,8 +2778,9 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction() def clear_transaction(self): - self.has_transaction_ = 0; - if self.transaction_ is not None: self.transaction_.Clear() + if self.has_transaction_: + self.has_transaction_ = 0; + if self.transaction_ is not None: self.transaction_.Clear() def has_transaction(self): return self.has_transaction_ @@ -1612,12 +2800,26 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): def clear_composite_index(self): self.composite_index_ = [] + def trusted(self): return self.trusted_ + + def set_trusted(self, x): + self.has_trusted_ = 1 + self.trusted_ = x + + def clear_trusted(self): + if self.has_trusted_: + self.has_trusted_ = 0 + self.trusted_ = 0 + + def has_trusted(self): return self.has_trusted_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.entity_size()): self.add_entity().CopyFrom(x.entity(i)) if (x.has_transaction()): self.mutable_transaction().MergeFrom(x.transaction()) for i in xrange(x.composite_index_size()): self.add_composite_index().CopyFrom(x.composite_index(i)) + if (x.has_trusted()): self.set_trusted(x.trusted()) def Equals(self, x): if x is self: return 1 @@ -1629,6 +2831,8 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): if len(self.composite_index_) != len(x.composite_index_): return 0 for e1, e2 in zip(self.composite_index_, x.composite_index_): if e1 != e2: return 0 + if self.has_trusted_ != x.has_trusted_: return 0 + if self.has_trusted_ and self.trusted_ != x.trusted_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -1647,12 +2851,14 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): if (self.has_transaction_): n += 1 + self.lengthString(self.transaction_.ByteSize()) n += 1 * len(self.composite_index_) for i in xrange(len(self.composite_index_)): n += self.lengthString(self.composite_index_[i].ByteSize()) + if (self.has_trusted_): n += 2 return n + 0 def Clear(self): self.clear_entity() self.clear_transaction() self.clear_composite_index() + self.clear_trusted() def OutputUnchecked(self, out): for i in xrange(len(self.entity_)): @@ -1667,6 +2873,9 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): out.putVarInt32(26) out.putVarInt32(self.composite_index_[i].ByteSize()) self.composite_index_[i].OutputUnchecked(out) + if (self.has_trusted_): + out.putVarInt32(32) + out.putBoolean(self.trusted_) def TryMerge(self, d): while d.avail() > 0: @@ -1689,6 +2898,9 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): d.skip(length) self.add_composite_index().TryMerge(tmp) continue + if tt == 32: + self.set_trusted(d.getBoolean()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -1715,28 +2927,33 @@ class PutRequest(ProtocolBuffer.ProtocolMessage): res+=e.__str__(prefix + " ", printElemNumber) res+=prefix+">\n" cnt+=1 + if self.has_trusted_: res+=prefix+("trusted: %s\n" % self.DebugFormatBool(self.trusted_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kentity = 1 ktransaction = 2 kcomposite_index = 3 - - _TEXT = ( - "ErrorCode", - "entity", - "transaction", - "composite_index", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ) + ktrusted = 4 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "entity", + 2: "transaction", + 3: "composite_index", + 4: "trusted", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1777,8 +2994,9 @@ class PutResponse(ProtocolBuffer.ProtocolMessage): def mutable_cost(self): self.has_cost_ = 1; return self.cost() def clear_cost(self): - self.has_cost_ = 0; - if self.cost_ is not None: self.cost_.Clear() + if self.has_cost_: + self.has_cost_ = 0; + if self.cost_ is not None: self.cost_.Clear() def has_cost(self): return self.has_cost_ @@ -1860,28 +3078,32 @@ class PutResponse(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res - kkey = 1 - kcost = 2 - _TEXT = ( - "ErrorCode", - "key", - "cost", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + kkey = 1 + kcost = 2 - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "key", + 2: "cost", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" class DeleteRequest(ProtocolBuffer.ProtocolMessage): has_transaction_ = 0 transaction_ = None + has_trusted_ = 0 + trusted_ = 0 def __init__(self, contents=None): self.key_ = [] @@ -1916,16 +3138,31 @@ class DeleteRequest(ProtocolBuffer.ProtocolMessage): def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction() def clear_transaction(self): - self.has_transaction_ = 0; - if self.transaction_ is not None: self.transaction_.Clear() + if self.has_transaction_: + self.has_transaction_ = 0; + if self.transaction_ is not None: self.transaction_.Clear() def has_transaction(self): return self.has_transaction_ + def trusted(self): return self.trusted_ + + def set_trusted(self, x): + self.has_trusted_ = 1 + self.trusted_ = x + + def clear_trusted(self): + if self.has_trusted_: + self.has_trusted_ = 0 + self.trusted_ = 0 + + def has_trusted(self): return self.has_trusted_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.key_size()): self.add_key().CopyFrom(x.key(i)) if (x.has_transaction()): self.mutable_transaction().MergeFrom(x.transaction()) + if (x.has_trusted()): self.set_trusted(x.trusted()) def Equals(self, x): if x is self: return 1 @@ -1934,6 +3171,8 @@ class DeleteRequest(ProtocolBuffer.ProtocolMessage): if e1 != e2: return 0 if self.has_transaction_ != x.has_transaction_: return 0 if self.has_transaction_ and self.transaction_ != x.transaction_: return 0 + if self.has_trusted_ != x.has_trusted_: return 0 + if self.has_trusted_ and self.trusted_ != x.trusted_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -1948,13 +3187,18 @@ class DeleteRequest(ProtocolBuffer.ProtocolMessage): n += 1 * len(self.key_) for i in xrange(len(self.key_)): n += self.lengthString(self.key_[i].ByteSize()) if (self.has_transaction_): n += 1 + self.lengthString(self.transaction_.ByteSize()) + if (self.has_trusted_): n += 2 return n + 0 def Clear(self): self.clear_key() self.clear_transaction() + self.clear_trusted() def OutputUnchecked(self, out): + if (self.has_trusted_): + out.putVarInt32(32) + out.putBoolean(self.trusted_) if (self.has_transaction_): out.putVarInt32(42) out.putVarInt32(self.transaction_.ByteSize()) @@ -1967,6 +3211,9 @@ class DeleteRequest(ProtocolBuffer.ProtocolMessage): def TryMerge(self, d): while d.avail() > 0: tt = d.getVarInt32() + if tt == 32: + self.set_trusted(d.getBoolean()) + continue if tt == 42: length = d.getVarInt32() tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) @@ -1997,36 +3244,30 @@ class DeleteRequest(ProtocolBuffer.ProtocolMessage): res+=prefix+"transaction <\n" res+=self.transaction_.__str__(prefix + " ", printElemNumber) res+=prefix+">\n" + if self.has_trusted_: res+=prefix+("trusted: %s\n" % self.DebugFormatBool(self.trusted_)) return res - kkey = 6 - ktransaction = 5 - - _TEXT = ( - "ErrorCode", - None, - None, - None, - None, - "transaction", - "key", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - ProtocolBuffer.Encoder.STRING, + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - ProtocolBuffer.Encoder.STRING, - - ) + kkey = 6 + ktransaction = 5 + ktrusted = 4 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 4: "trusted", + 5: "transaction", + 6: "key", + }, 6) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.STRING, + 6: ProtocolBuffer.Encoder.STRING, + }, 6, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -2050,8 +3291,9 @@ class DeleteResponse(ProtocolBuffer.ProtocolMessage): def mutable_cost(self): self.has_cost_ = 1; return self.cost() def clear_cost(self): - self.has_cost_ = 0; - if self.cost_ is not None: self.cost_.Clear() + if self.has_cost_: + self.has_cost_ = 0; + if self.cost_ is not None: self.cost_.Clear() def has_cost(self): return self.has_cost_ @@ -2106,25 +3348,30 @@ class DeleteResponse(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res - kcost = 1 - _TEXT = ( - "ErrorCode", - "cost", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kcost = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "cost", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" class NextRequest(ProtocolBuffer.ProtocolMessage): has_cursor_ = 0 has_count_ = 0 - count_ = 1 + count_ = 0 + has_compile_ = 0 + compile_ = 0 def __init__(self, contents=None): self.cursor_ = Cursor() @@ -2145,16 +3392,31 @@ class NextRequest(ProtocolBuffer.ProtocolMessage): self.count_ = x def clear_count(self): - self.has_count_ = 0 - self.count_ = 1 + if self.has_count_: + self.has_count_ = 0 + self.count_ = 0 def has_count(self): return self.has_count_ + def compile(self): return self.compile_ + + def set_compile(self, x): + self.has_compile_ = 1 + self.compile_ = x + + def clear_compile(self): + if self.has_compile_: + self.has_compile_ = 0 + self.compile_ = 0 + + def has_compile(self): return self.has_compile_ + def MergeFrom(self, x): assert x is not self if (x.has_cursor()): self.mutable_cursor().MergeFrom(x.cursor()) if (x.has_count()): self.set_count(x.count()) + if (x.has_compile()): self.set_compile(x.compile()) def Equals(self, x): if x is self: return 1 @@ -2162,6 +3424,8 @@ class NextRequest(ProtocolBuffer.ProtocolMessage): if self.has_cursor_ and self.cursor_ != x.cursor_: return 0 if self.has_count_ != x.has_count_: return 0 if self.has_count_ and self.count_ != x.count_: return 0 + if self.has_compile_ != x.has_compile_: return 0 + if self.has_compile_ and self.compile_ != x.compile_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -2177,11 +3441,13 @@ class NextRequest(ProtocolBuffer.ProtocolMessage): n = 0 n += self.lengthString(self.cursor_.ByteSize()) if (self.has_count_): n += 1 + self.lengthVarInt64(self.count_) + if (self.has_compile_): n += 2 return n + 1 def Clear(self): self.clear_cursor() self.clear_count() + self.clear_compile() def OutputUnchecked(self, out): out.putVarInt32(10) @@ -2190,6 +3456,9 @@ class NextRequest(ProtocolBuffer.ProtocolMessage): if (self.has_count_): out.putVarInt32(16) out.putVarInt32(self.count_) + if (self.has_compile_): + out.putVarInt32(24) + out.putBoolean(self.compile_) def TryMerge(self, d): while d.avail() > 0: @@ -2203,6 +3472,9 @@ class NextRequest(ProtocolBuffer.ProtocolMessage): if tt == 16: self.set_count(d.getVarInt32()) continue + if tt == 24: + self.set_compile(d.getBoolean()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -2214,24 +3486,30 @@ class NextRequest(ProtocolBuffer.ProtocolMessage): res+=self.cursor_.__str__(prefix + " ", printElemNumber) res+=prefix+">\n" if self.has_count_: res+=prefix+("count: %s\n" % self.DebugFormatInt32(self.count_)) + if self.has_compile_: res+=prefix+("compile: %s\n" % self.DebugFormatBool(self.compile_)) return res - kcursor = 1 - kcount = 2 - - _TEXT = ( - "ErrorCode", - "cursor", - "count", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - ProtocolBuffer.Encoder.NUMERIC, + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - ) + kcursor = 1 + kcount = 2 + kcompile = 3 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "cursor", + 2: "count", + 3: "compile", + }, 3) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.NUMERIC, + }, 3, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -2240,6 +3518,10 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): cursor_ = None has_more_results_ = 0 more_results_ = 0 + has_keys_only_ = 0 + keys_only_ = 0 + has_compiled_query_ = 0 + compiled_query_ = None def __init__(self, contents=None): self.result_ = [] @@ -2258,8 +3540,9 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): def mutable_cursor(self): self.has_cursor_ = 1; return self.cursor() def clear_cursor(self): - self.has_cursor_ = 0; - if self.cursor_ is not None: self.cursor_.Clear() + if self.has_cursor_: + self.has_cursor_ = 0; + if self.cursor_ is not None: self.cursor_.Clear() def has_cursor(self): return self.has_cursor_ @@ -2286,17 +3569,51 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): self.more_results_ = x def clear_more_results(self): - self.has_more_results_ = 0 - self.more_results_ = 0 + if self.has_more_results_: + self.has_more_results_ = 0 + self.more_results_ = 0 def has_more_results(self): return self.has_more_results_ + def keys_only(self): return self.keys_only_ + + def set_keys_only(self, x): + self.has_keys_only_ = 1 + self.keys_only_ = x + + def clear_keys_only(self): + if self.has_keys_only_: + self.has_keys_only_ = 0 + self.keys_only_ = 0 + + def has_keys_only(self): return self.has_keys_only_ + + def compiled_query(self): + if self.compiled_query_ is None: + self.lazy_init_lock_.acquire() + try: + if self.compiled_query_ is None: self.compiled_query_ = CompiledQuery() + finally: + self.lazy_init_lock_.release() + return self.compiled_query_ + + def mutable_compiled_query(self): self.has_compiled_query_ = 1; return self.compiled_query() + + def clear_compiled_query(self): + if self.has_compiled_query_: + self.has_compiled_query_ = 0; + if self.compiled_query_ is not None: self.compiled_query_.Clear() + + def has_compiled_query(self): return self.has_compiled_query_ + def MergeFrom(self, x): assert x is not self if (x.has_cursor()): self.mutable_cursor().MergeFrom(x.cursor()) for i in xrange(x.result_size()): self.add_result().CopyFrom(x.result(i)) if (x.has_more_results()): self.set_more_results(x.more_results()) + if (x.has_keys_only()): self.set_keys_only(x.keys_only()) + if (x.has_compiled_query()): self.mutable_compiled_query().MergeFrom(x.compiled_query()) def Equals(self, x): if x is self: return 1 @@ -2307,6 +3624,10 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): if e1 != e2: return 0 if self.has_more_results_ != x.has_more_results_: return 0 if self.has_more_results_ and self.more_results_ != x.more_results_: return 0 + if self.has_keys_only_ != x.has_keys_only_: return 0 + if self.has_keys_only_ and self.keys_only_ != x.keys_only_: return 0 + if self.has_compiled_query_ != x.has_compiled_query_: return 0 + if self.has_compiled_query_ and self.compiled_query_ != x.compiled_query_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -2318,6 +3639,7 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): initialized = 0 if debug_strs is not None: debug_strs.append('Required field: more_results not set.') + if (self.has_compiled_query_ and not self.compiled_query_.IsInitialized(debug_strs)): initialized = 0 return initialized def ByteSize(self): @@ -2325,12 +3647,16 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): if (self.has_cursor_): n += 1 + self.lengthString(self.cursor_.ByteSize()) n += 1 * len(self.result_) for i in xrange(len(self.result_)): n += self.lengthString(self.result_[i].ByteSize()) + if (self.has_keys_only_): n += 2 + if (self.has_compiled_query_): n += 1 + self.lengthString(self.compiled_query_.ByteSize()) return n + 2 def Clear(self): self.clear_cursor() self.clear_result() self.clear_more_results() + self.clear_keys_only() + self.clear_compiled_query() def OutputUnchecked(self, out): if (self.has_cursor_): @@ -2343,6 +3669,13 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): self.result_[i].OutputUnchecked(out) out.putVarInt32(24) out.putBoolean(self.more_results_) + if (self.has_keys_only_): + out.putVarInt32(32) + out.putBoolean(self.keys_only_) + if (self.has_compiled_query_): + out.putVarInt32(42) + out.putVarInt32(self.compiled_query_.ByteSize()) + self.compiled_query_.OutputUnchecked(out) def TryMerge(self, d): while d.avail() > 0: @@ -2362,6 +3695,15 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): if tt == 24: self.set_more_results(d.getBoolean()) continue + if tt == 32: + self.set_keys_only(d.getBoolean()) + continue + if tt == 42: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_compiled_query().TryMerge(tmp) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -2381,32 +3723,220 @@ class QueryResult(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" cnt+=1 if self.has_more_results_: res+=prefix+("more_results: %s\n" % self.DebugFormatBool(self.more_results_)) + if self.has_keys_only_: res+=prefix+("keys_only: %s\n" % self.DebugFormatBool(self.keys_only_)) + if self.has_compiled_query_: + res+=prefix+"compiled_query <\n" + res+=self.compiled_query_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kcursor = 1 kresult = 2 kmore_results = 3 + kkeys_only = 4 + kcompiled_query = 5 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "cursor", + 2: "result", + 3: "more_results", + 4: "keys_only", + 5: "compiled_query", + }, 5) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.STRING, + }, 5, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class GetSchemaRequest(ProtocolBuffer.ProtocolMessage): + has_app_ = 0 + app_ = "" + has_start_kind_ = 0 + start_kind_ = "" + has_end_kind_ = 0 + end_kind_ = "" + has_properties_ = 0 + properties_ = 1 + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def app(self): return self.app_ + + def set_app(self, x): + self.has_app_ = 1 + self.app_ = x + + def clear_app(self): + if self.has_app_: + self.has_app_ = 0 + self.app_ = "" + + def has_app(self): return self.has_app_ + + def start_kind(self): return self.start_kind_ - _TEXT = ( - "ErrorCode", - "cursor", - "result", - "more_results", - ) + def set_start_kind(self, x): + self.has_start_kind_ = 1 + self.start_kind_ = x - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + def clear_start_kind(self): + if self.has_start_kind_: + self.has_start_kind_ = 0 + self.start_kind_ = "" - ProtocolBuffer.Encoder.STRING, + def has_start_kind(self): return self.has_start_kind_ - ProtocolBuffer.Encoder.NUMERIC, + def end_kind(self): return self.end_kind_ - ) + def set_end_kind(self, x): + self.has_end_kind_ = 1 + self.end_kind_ = x + + def clear_end_kind(self): + if self.has_end_kind_: + self.has_end_kind_ = 0 + self.end_kind_ = "" + + def has_end_kind(self): return self.has_end_kind_ + + def properties(self): return self.properties_ + + def set_properties(self, x): + self.has_properties_ = 1 + self.properties_ = x + + def clear_properties(self): + if self.has_properties_: + self.has_properties_ = 0 + self.properties_ = 1 + + def has_properties(self): return self.has_properties_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_app()): self.set_app(x.app()) + if (x.has_start_kind()): self.set_start_kind(x.start_kind()) + if (x.has_end_kind()): self.set_end_kind(x.end_kind()) + if (x.has_properties()): self.set_properties(x.properties()) + + def Equals(self, x): + if x is self: return 1 + if self.has_app_ != x.has_app_: return 0 + if self.has_app_ and self.app_ != x.app_: return 0 + if self.has_start_kind_ != x.has_start_kind_: return 0 + if self.has_start_kind_ and self.start_kind_ != x.start_kind_: return 0 + if self.has_end_kind_ != x.has_end_kind_: return 0 + if self.has_end_kind_ and self.end_kind_ != x.end_kind_: return 0 + if self.has_properties_ != x.has_properties_: return 0 + if self.has_properties_ and self.properties_ != x.properties_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_app_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: app not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(len(self.app_)) + if (self.has_start_kind_): n += 1 + self.lengthString(len(self.start_kind_)) + if (self.has_end_kind_): n += 1 + self.lengthString(len(self.end_kind_)) + if (self.has_properties_): n += 2 + return n + 1 + + def Clear(self): + self.clear_app() + self.clear_start_kind() + self.clear_end_kind() + self.clear_properties() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putPrefixedString(self.app_) + if (self.has_start_kind_): + out.putVarInt32(18) + out.putPrefixedString(self.start_kind_) + if (self.has_end_kind_): + out.putVarInt32(26) + out.putPrefixedString(self.end_kind_) + if (self.has_properties_): + out.putVarInt32(32) + out.putBoolean(self.properties_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + self.set_app(d.getPrefixedString()) + continue + if tt == 18: + self.set_start_kind(d.getPrefixedString()) + continue + if tt == 26: + self.set_end_kind(d.getPrefixedString()) + continue + if tt == 32: + self.set_properties(d.getBoolean()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_app_: res+=prefix+("app: %s\n" % self.DebugFormatString(self.app_)) + if self.has_start_kind_: res+=prefix+("start_kind: %s\n" % self.DebugFormatString(self.start_kind_)) + if self.has_end_kind_: res+=prefix+("end_kind: %s\n" % self.DebugFormatString(self.end_kind_)) + if self.has_properties_: res+=prefix+("properties: %s\n" % self.DebugFormatBool(self.properties_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kapp = 1 + kstart_kind = 2 + kend_kind = 3 + kproperties = 4 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "app", + 2: "start_kind", + 3: "end_kind", + 4: "properties", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" class Schema(ProtocolBuffer.ProtocolMessage): + has_more_results_ = 0 + more_results_ = 0 def __init__(self, contents=None): self.kind_ = [] @@ -2428,16 +3958,32 @@ class Schema(ProtocolBuffer.ProtocolMessage): def clear_kind(self): self.kind_ = [] + def more_results(self): return self.more_results_ + + def set_more_results(self, x): + self.has_more_results_ = 1 + self.more_results_ = x + + def clear_more_results(self): + if self.has_more_results_: + self.has_more_results_ = 0 + self.more_results_ = 0 + + def has_more_results(self): return self.has_more_results_ + def MergeFrom(self, x): assert x is not self for i in xrange(x.kind_size()): self.add_kind().CopyFrom(x.kind(i)) + if (x.has_more_results()): self.set_more_results(x.more_results()) def Equals(self, x): if x is self: return 1 if len(self.kind_) != len(x.kind_): return 0 for e1, e2 in zip(self.kind_, x.kind_): if e1 != e2: return 0 + if self.has_more_results_ != x.has_more_results_: return 0 + if self.has_more_results_ and self.more_results_ != x.more_results_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -2450,16 +3996,21 @@ class Schema(ProtocolBuffer.ProtocolMessage): n = 0 n += 1 * len(self.kind_) for i in xrange(len(self.kind_)): n += self.lengthString(self.kind_[i].ByteSize()) + if (self.has_more_results_): n += 2 return n + 0 def Clear(self): self.clear_kind() + self.clear_more_results() def OutputUnchecked(self, out): for i in xrange(len(self.kind_)): out.putVarInt32(10) out.putVarInt32(self.kind_[i].ByteSize()) self.kind_[i].OutputUnchecked(out) + if (self.has_more_results_): + out.putVarInt32(16) + out.putBoolean(self.more_results_) def TryMerge(self, d): while d.avail() > 0: @@ -2470,6 +4021,9 @@ class Schema(ProtocolBuffer.ProtocolMessage): d.skip(length) self.add_kind().TryMerge(tmp) continue + if tt == 16: + self.set_more_results(d.getBoolean()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -2484,20 +4038,264 @@ class Schema(ProtocolBuffer.ProtocolMessage): res+=e.__str__(prefix + " ", printElemNumber) res+=prefix+">\n" cnt+=1 + if self.has_more_results_: res+=prefix+("more_results: %s\n" % self.DebugFormatBool(self.more_results_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kkind = 1 + kmore_results = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "kind", + 2: "more_results", + }, 2) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.NUMERIC, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class AllocateIdsRequest(ProtocolBuffer.ProtocolMessage): + has_model_key_ = 0 + has_size_ = 0 + size_ = 0 + + def __init__(self, contents=None): + self.model_key_ = Reference() + if contents is not None: self.MergeFromString(contents) - _TEXT = ( - "ErrorCode", - "kind", - ) + def model_key(self): return self.model_key_ - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + def mutable_model_key(self): self.has_model_key_ = 1; return self.model_key_ - ) + def clear_model_key(self):self.has_model_key_ = 0; self.model_key_.Clear() + + def has_model_key(self): return self.has_model_key_ + + def size(self): return self.size_ + + def set_size(self, x): + self.has_size_ = 1 + self.size_ = x + + def clear_size(self): + if self.has_size_: + self.has_size_ = 0 + self.size_ = 0 + + def has_size(self): return self.has_size_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_model_key()): self.mutable_model_key().MergeFrom(x.model_key()) + if (x.has_size()): self.set_size(x.size()) + + def Equals(self, x): + if x is self: return 1 + if self.has_model_key_ != x.has_model_key_: return 0 + if self.has_model_key_ and self.model_key_ != x.model_key_: return 0 + if self.has_size_ != x.has_size_: return 0 + if self.has_size_ and self.size_ != x.size_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_model_key_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: model_key not set.') + elif not self.model_key_.IsInitialized(debug_strs): initialized = 0 + if (not self.has_size_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: size not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(self.model_key_.ByteSize()) + n += self.lengthVarInt64(self.size_) + return n + 2 + + def Clear(self): + self.clear_model_key() + self.clear_size() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putVarInt32(self.model_key_.ByteSize()) + self.model_key_.OutputUnchecked(out) + out.putVarInt32(16) + out.putVarInt64(self.size_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_model_key().TryMerge(tmp) + continue + if tt == 16: + self.set_size(d.getVarInt64()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_model_key_: + res+=prefix+"model_key <\n" + res+=self.model_key_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + if self.has_size_: res+=prefix+("size: %s\n" % self.DebugFormatInt64(self.size_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kmodel_key = 1 + ksize = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "model_key", + 2: "size", + }, 2) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.NUMERIC, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class AllocateIdsResponse(ProtocolBuffer.ProtocolMessage): + has_start_ = 0 + start_ = 0 + has_end_ = 0 + end_ = 0 + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def start(self): return self.start_ + + def set_start(self, x): + self.has_start_ = 1 + self.start_ = x + + def clear_start(self): + if self.has_start_: + self.has_start_ = 0 + self.start_ = 0 + + def has_start(self): return self.has_start_ + + def end(self): return self.end_ + + def set_end(self, x): + self.has_end_ = 1 + self.end_ = x + + def clear_end(self): + if self.has_end_: + self.has_end_ = 0 + self.end_ = 0 + + def has_end(self): return self.has_end_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_start()): self.set_start(x.start()) + if (x.has_end()): self.set_end(x.end()) + + def Equals(self, x): + if x is self: return 1 + if self.has_start_ != x.has_start_: return 0 + if self.has_start_ and self.start_ != x.start_: return 0 + if self.has_end_ != x.has_end_: return 0 + if self.has_end_ and self.end_ != x.end_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_start_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: start not set.') + if (not self.has_end_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: end not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthVarInt64(self.start_) + n += self.lengthVarInt64(self.end_) + return n + 2 + + def Clear(self): + self.clear_start() + self.clear_end() + + def OutputUnchecked(self, out): + out.putVarInt32(8) + out.putVarInt64(self.start_) + out.putVarInt32(16) + out.putVarInt64(self.end_) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 8: + self.set_start(d.getVarInt64()) + continue + if tt == 16: + self.set_end(d.getVarInt64()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_start_: res+=prefix+("start: %s\n" % self.DebugFormatInt64(self.start_)) + if self.has_end_: res+=prefix+("end: %s\n" % self.DebugFormatInt64(self.end_)) + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kstart = 1 + kend = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "start", + 2: "end", + }, 2) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -2581,20 +4379,295 @@ class CompositeIndices(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kindex = 1 - _TEXT = ( - "ErrorCode", - "index", - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "index", + }, 1) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ActionRequest(ProtocolBuffer.ProtocolMessage): + has_transaction_ = 0 + has_action_ = 0 + + def __init__(self, contents=None): + self.transaction_ = Transaction() + self.action_ = Action() + if contents is not None: self.MergeFromString(contents) + + def transaction(self): return self.transaction_ + + def mutable_transaction(self): self.has_transaction_ = 1; return self.transaction_ + + def clear_transaction(self):self.has_transaction_ = 0; self.transaction_.Clear() + + def has_transaction(self): return self.has_transaction_ + + def action(self): return self.action_ + + def mutable_action(self): self.has_action_ = 1; return self.action_ + + def clear_action(self):self.has_action_ = 0; self.action_.Clear() + + def has_action(self): return self.has_action_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_transaction()): self.mutable_transaction().MergeFrom(x.transaction()) + if (x.has_action()): self.mutable_action().MergeFrom(x.action()) + + def Equals(self, x): + if x is self: return 1 + if self.has_transaction_ != x.has_transaction_: return 0 + if self.has_transaction_ and self.transaction_ != x.transaction_: return 0 + if self.has_action_ != x.has_action_: return 0 + if self.has_action_ and self.action_ != x.action_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_transaction_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: transaction not set.') + elif not self.transaction_.IsInitialized(debug_strs): initialized = 0 + if (not self.has_action_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: action not set.') + elif not self.action_.IsInitialized(debug_strs): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthString(self.transaction_.ByteSize()) + n += self.lengthString(self.action_.ByteSize()) + return n + 2 + + def Clear(self): + self.clear_transaction() + self.clear_action() + + def OutputUnchecked(self, out): + out.putVarInt32(10) + out.putVarInt32(self.transaction_.ByteSize()) + self.transaction_.OutputUnchecked(out) + out.putVarInt32(18) + out.putVarInt32(self.action_.ByteSize()) + self.action_.OutputUnchecked(out) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_transaction().TryMerge(tmp) + continue + if tt == 18: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_action().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_transaction_: + res+=prefix+"transaction <\n" + res+=self.transaction_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + if self.has_action_: + res+=prefix+"action <\n" + res+=self.action_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + ktransaction = 1 + kaction = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "transaction", + 2: "action", + }, 2) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ActionResponse(ProtocolBuffer.ProtocolMessage): + + def __init__(self, contents=None): + pass + if contents is not None: self.MergeFromString(contents) + + + def MergeFrom(self, x): + assert x is not self + + def Equals(self, x): + if x is self: return 1 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + return initialized + + def ByteSize(self): + n = 0 + return n + 0 + + def Clear(self): + pass + + def OutputUnchecked(self, out): + pass + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + }, 0) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + }, 0, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class CommitResponse(ProtocolBuffer.ProtocolMessage): + has_cost_ = 0 + cost_ = None + + def __init__(self, contents=None): + self.lazy_init_lock_ = thread.allocate_lock() + if contents is not None: self.MergeFromString(contents) + + def cost(self): + if self.cost_ is None: + self.lazy_init_lock_.acquire() + try: + if self.cost_ is None: self.cost_ = Cost() + finally: + self.lazy_init_lock_.release() + return self.cost_ + + def mutable_cost(self): self.has_cost_ = 1; return self.cost() + + def clear_cost(self): + if self.has_cost_: + self.has_cost_ = 0; + if self.cost_ is not None: self.cost_.Clear() + + def has_cost(self): return self.has_cost_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_cost()): self.mutable_cost().MergeFrom(x.cost()) + + def Equals(self, x): + if x is self: return 1 + if self.has_cost_ != x.has_cost_: return 0 + if self.has_cost_ and self.cost_ != x.cost_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (self.has_cost_ and not self.cost_.IsInitialized(debug_strs)): initialized = 0 + return initialized + + def ByteSize(self): + n = 0 + if (self.has_cost_): n += 1 + self.lengthString(self.cost_.ByteSize()) + return n + 0 + + def Clear(self): + self.clear_cost() + + def OutputUnchecked(self, out): + if (self.has_cost_): + out.putVarInt32(10) + out.putVarInt32(self.cost_.ByteSize()) + self.cost_.OutputUnchecked(out) + + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 10: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_cost().TryMerge(tmp) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) + + + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_cost_: + res+=prefix+"cost <\n" + res+=self.cost_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + return res + + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kcost = 1 - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "cost", + }, 1) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + }, 1, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" -__all__ = ['Transaction','Query','Query_Filter','Query_Order','QueryExplanation','Cursor','Error','Cost','GetRequest','GetResponse','GetResponse_Entity','PutRequest','PutResponse','DeleteRequest','DeleteResponse','NextRequest','QueryResult','Schema','CompositeIndices'] +__all__ = ['Transaction','Query','Query_Filter','Query_Order','CompiledQuery','CompiledQuery_PrimaryScan','CompiledQuery_MergeJoinScan','CompiledQuery_EntityFilter','RunCompiledQueryRequest','QueryExplanation','Cursor','Error','Cost','GetRequest','GetResponse','GetResponse_Entity','PutRequest','PutResponse','DeleteRequest','DeleteResponse','NextRequest','QueryResult','GetSchemaRequest','Schema','AllocateIdsRequest','AllocateIdsResponse','CompositeIndices','ActionRequest','ActionResponse','CommitResponse'] google_appengine/google/appengine/datastore/datastore_pb.py @@ -40,8 +40,9 @@ class PropertyValue_ReferenceValuePathElement(ProtocolBuffer.ProtocolMessage): self.type_ = x def clear_type(self): - self.has_type_ = 0 - self.type_ = "" + if self.has_type_: + self.has_type_ = 0 + self.type_ = "" def has_type(self): return self.has_type_ @@ -52,8 +53,9 @@ class PropertyValue_ReferenceValuePathElement(ProtocolBuffer.ProtocolMessage): self.id_ = x def clear_id(self): - self.has_id_ = 0 - self.id_ = 0 + if self.has_id_: + self.has_id_ = 0 + self.id_ = 0 def has_id(self): return self.has_id_ @@ -64,8 +66,9 @@ class PropertyValue_ReferenceValuePathElement(ProtocolBuffer.ProtocolMessage): self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -156,8 +159,9 @@ class PropertyValue_PointValue(ProtocolBuffer.ProtocolMessage): self.x_ = x def clear_x(self): - self.has_x_ = 0 - self.x_ = 0.0 + if self.has_x_: + self.has_x_ = 0 + self.x_ = 0.0 def has_x(self): return self.has_x_ @@ -168,8 +172,9 @@ class PropertyValue_PointValue(ProtocolBuffer.ProtocolMessage): self.y_ = x def clear_y(self): - self.has_y_ = 0 - self.y_ = 0.0 + if self.has_y_: + self.has_y_ = 0 + self.y_ = 0.0 def has_y(self): return self.has_y_ @@ -242,6 +247,8 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): nickname_ = "" has_gaiaid_ = 0 gaiaid_ = 0 + has_obfuscated_gaiaid_ = 0 + obfuscated_gaiaid_ = "" def __init__(self, contents=None): if contents is not None: self.MergeFromString(contents) @@ -253,8 +260,9 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): self.email_ = x def clear_email(self): - self.has_email_ = 0 - self.email_ = "" + if self.has_email_: + self.has_email_ = 0 + self.email_ = "" def has_email(self): return self.has_email_ @@ -265,8 +273,9 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): self.auth_domain_ = x def clear_auth_domain(self): - self.has_auth_domain_ = 0 - self.auth_domain_ = "" + if self.has_auth_domain_: + self.has_auth_domain_ = 0 + self.auth_domain_ = "" def has_auth_domain(self): return self.has_auth_domain_ @@ -277,8 +286,9 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): self.nickname_ = x def clear_nickname(self): - self.has_nickname_ = 0 - self.nickname_ = "" + if self.has_nickname_: + self.has_nickname_ = 0 + self.nickname_ = "" def has_nickname(self): return self.has_nickname_ @@ -289,11 +299,25 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): self.gaiaid_ = x def clear_gaiaid(self): - self.has_gaiaid_ = 0 - self.gaiaid_ = 0 + if self.has_gaiaid_: + self.has_gaiaid_ = 0 + self.gaiaid_ = 0 def has_gaiaid(self): return self.has_gaiaid_ + def obfuscated_gaiaid(self): return self.obfuscated_gaiaid_ + + def set_obfuscated_gaiaid(self, x): + self.has_obfuscated_gaiaid_ = 1 + self.obfuscated_gaiaid_ = x + + def clear_obfuscated_gaiaid(self): + if self.has_obfuscated_gaiaid_: + self.has_obfuscated_gaiaid_ = 0 + self.obfuscated_gaiaid_ = "" + + def has_obfuscated_gaiaid(self): return self.has_obfuscated_gaiaid_ + def MergeFrom(self, x): assert x is not self @@ -301,6 +325,7 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain()) if (x.has_nickname()): self.set_nickname(x.nickname()) if (x.has_gaiaid()): self.set_gaiaid(x.gaiaid()) + if (x.has_obfuscated_gaiaid()): self.set_obfuscated_gaiaid(x.obfuscated_gaiaid()) def Equals(self, x): if x is self: return 1 @@ -312,6 +337,8 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): if self.has_nickname_ and self.nickname_ != x.nickname_: return 0 if self.has_gaiaid_ != x.has_gaiaid_: return 0 if self.has_gaiaid_ and self.gaiaid_ != x.gaiaid_: return 0 + if self.has_obfuscated_gaiaid_ != x.has_obfuscated_gaiaid_: return 0 + if self.has_obfuscated_gaiaid_ and self.obfuscated_gaiaid_ != x.obfuscated_gaiaid_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -336,6 +363,7 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): n += self.lengthString(len(self.auth_domain_)) if (self.has_nickname_): n += 1 + self.lengthString(len(self.nickname_)) n += self.lengthVarInt64(self.gaiaid_) + if (self.has_obfuscated_gaiaid_): n += 2 + self.lengthString(len(self.obfuscated_gaiaid_)) return n + 4 def Clear(self): @@ -343,6 +371,7 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): self.clear_auth_domain() self.clear_nickname() self.clear_gaiaid() + self.clear_obfuscated_gaiaid() def OutputUnchecked(self, out): out.putVarInt32(74) @@ -354,6 +383,9 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): out.putPrefixedString(self.nickname_) out.putVarInt32(144) out.putVarInt64(self.gaiaid_) + if (self.has_obfuscated_gaiaid_): + out.putVarInt32(154) + out.putPrefixedString(self.obfuscated_gaiaid_) def TryMerge(self, d): while 1: @@ -371,6 +403,9 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): if tt == 144: self.set_gaiaid(d.getVarInt64()) continue + if tt == 154: + self.set_obfuscated_gaiaid(d.getPrefixedString()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -381,6 +416,7 @@ class PropertyValue_UserValue(ProtocolBuffer.ProtocolMessage): if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_)) if self.has_nickname_: res+=prefix+("nickname: %s\n" % self.DebugFormatString(self.nickname_)) if self.has_gaiaid_: res+=prefix+("gaiaid: %s\n" % self.DebugFormatInt64(self.gaiaid_)) + if self.has_obfuscated_gaiaid_: res+=prefix+("obfuscated_gaiaid: %s\n" % self.DebugFormatString(self.obfuscated_gaiaid_)) return res class PropertyValue_ReferenceValue(ProtocolBuffer.ProtocolMessage): @@ -398,8 +434,9 @@ class PropertyValue_ReferenceValue(ProtocolBuffer.ProtocolMessage): self.app_ = x def clear_app(self): - self.has_app_ = 0 - self.app_ = "" + if self.has_app_: + self.has_app_ = 0 + self.app_ = "" def has_app(self): return self.has_app_ @@ -517,8 +554,9 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): self.int64value_ = x def clear_int64value(self): - self.has_int64value_ = 0 - self.int64value_ = 0 + if self.has_int64value_: + self.has_int64value_ = 0 + self.int64value_ = 0 def has_int64value(self): return self.has_int64value_ @@ -529,8 +567,9 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): self.booleanvalue_ = x def clear_booleanvalue(self): - self.has_booleanvalue_ = 0 - self.booleanvalue_ = 0 + if self.has_booleanvalue_: + self.has_booleanvalue_ = 0 + self.booleanvalue_ = 0 def has_booleanvalue(self): return self.has_booleanvalue_ @@ -541,8 +580,9 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): self.stringvalue_ = x def clear_stringvalue(self): - self.has_stringvalue_ = 0 - self.stringvalue_ = "" + if self.has_stringvalue_: + self.has_stringvalue_ = 0 + self.stringvalue_ = "" def has_stringvalue(self): return self.has_stringvalue_ @@ -553,8 +593,9 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): self.doublevalue_ = x def clear_doublevalue(self): - self.has_doublevalue_ = 0 - self.doublevalue_ = 0.0 + if self.has_doublevalue_: + self.has_doublevalue_ = 0 + self.doublevalue_ = 0.0 def has_doublevalue(self): return self.has_doublevalue_ @@ -570,8 +611,9 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): def mutable_pointvalue(self): self.has_pointvalue_ = 1; return self.pointvalue() def clear_pointvalue(self): - self.has_pointvalue_ = 0; - if self.pointvalue_ is not None: self.pointvalue_.Clear() + if self.has_pointvalue_: + self.has_pointvalue_ = 0; + if self.pointvalue_ is not None: self.pointvalue_.Clear() def has_pointvalue(self): return self.has_pointvalue_ @@ -587,8 +629,9 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): def mutable_uservalue(self): self.has_uservalue_ = 1; return self.uservalue() def clear_uservalue(self): - self.has_uservalue_ = 0; - if self.uservalue_ is not None: self.uservalue_.Clear() + if self.has_uservalue_: + self.has_uservalue_ = 0; + if self.uservalue_ is not None: self.uservalue_.Clear() def has_uservalue(self): return self.has_uservalue_ @@ -604,8 +647,9 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): def mutable_referencevalue(self): self.has_referencevalue_ = 1; return self.referencevalue() def clear_referencevalue(self): - self.has_referencevalue_ = 0; - if self.referencevalue_ is not None: self.referencevalue_.Clear() + if self.has_referencevalue_: + self.has_referencevalue_ = 0; + if self.referencevalue_ is not None: self.referencevalue_.Clear() def has_referencevalue(self): return self.has_referencevalue_ @@ -739,6 +783,10 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): res+=prefix+"}\n" return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kint64Value = 1 kbooleanValue = 2 kstringValue = 3 @@ -751,6 +799,7 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): kUserValueauth_domain = 10 kUserValuenickname = 11 kUserValuegaiaid = 18 + kUserValueobfuscated_gaiaid = 19 kReferenceValueGroup = 12 kReferenceValueapp = 13 kReferenceValuePathElementGroup = 14 @@ -758,67 +807,51 @@ class PropertyValue(ProtocolBuffer.ProtocolMessage): kReferenceValuePathElementid = 16 kReferenceValuePathElementname = 17 - _TEXT = ( - "ErrorCode", - "int64Value", - "booleanValue", - "stringValue", - "doubleValue", - "PointValue", - "x", - "y", - "UserValue", - "email", - "auth_domain", - "nickname", - "ReferenceValue", - "app", - "PathElement", - "type", - "id", - "name", - "gaiaid", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.DOUBLE, - - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.DOUBLE, - - ProtocolBuffer.Encoder.DOUBLE, - - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "int64Value", + 2: "booleanValue", + 3: "stringValue", + 4: "doubleValue", + 5: "PointValue", + 6: "x", + 7: "y", + 8: "UserValue", + 9: "email", + 10: "auth_domain", + 11: "nickname", + 12: "ReferenceValue", + 13: "app", + 14: "PathElement", + 15: "type", + 16: "id", + 17: "name", + 18: "gaiaid", + 19: "obfuscated_gaiaid", + }, 19) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.DOUBLE, + 5: ProtocolBuffer.Encoder.STARTGROUP, + 6: ProtocolBuffer.Encoder.DOUBLE, + 7: ProtocolBuffer.Encoder.DOUBLE, + 8: ProtocolBuffer.Encoder.STARTGROUP, + 9: ProtocolBuffer.Encoder.STRING, + 10: ProtocolBuffer.Encoder.STRING, + 11: ProtocolBuffer.Encoder.STRING, + 12: ProtocolBuffer.Encoder.STARTGROUP, + 13: ProtocolBuffer.Encoder.STRING, + 14: ProtocolBuffer.Encoder.STARTGROUP, + 15: ProtocolBuffer.Encoder.STRING, + 16: ProtocolBuffer.Encoder.NUMERIC, + 17: ProtocolBuffer.Encoder.STRING, + 18: ProtocolBuffer.Encoder.NUMERIC, + 19: ProtocolBuffer.Encoder.STRING, + }, 19, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -840,6 +873,7 @@ class Property(ProtocolBuffer.ProtocolMessage): GD_PHONENUMBER = 11 GD_POSTALADDRESS = 12 GD_RATING = 13 + BLOBKEY = 17 _Meaning_NAMES = { 14: "BLOB", @@ -858,6 +892,7 @@ class Property(ProtocolBuffer.ProtocolMessage): 11: "GD_PHONENUMBER", 12: "GD_POSTALADDRESS", 13: "GD_RATING", + 17: "BLOBKEY", } def Meaning_Name(cls, x): return cls._Meaning_NAMES.get(x, "") @@ -884,8 +919,9 @@ class Property(ProtocolBuffer.ProtocolMessage): self.meaning_ = x def clear_meaning(self): - self.has_meaning_ = 0 - self.meaning_ = 0 + if self.has_meaning_: + self.has_meaning_ = 0 + self.meaning_ = 0 def has_meaning(self): return self.has_meaning_ @@ -896,8 +932,9 @@ class Property(ProtocolBuffer.ProtocolMessage): self.meaning_uri_ = x def clear_meaning_uri(self): - self.has_meaning_uri_ = 0 - self.meaning_uri_ = "" + if self.has_meaning_uri_: + self.has_meaning_uri_ = 0 + self.meaning_uri_ = "" def has_meaning_uri(self): return self.has_meaning_uri_ @@ -908,8 +945,9 @@ class Property(ProtocolBuffer.ProtocolMessage): self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -928,8 +966,9 @@ class Property(ProtocolBuffer.ProtocolMessage): self.multiple_ = x def clear_multiple(self): - self.has_multiple_ = 0 - self.multiple_ = 0 + if self.has_multiple_: + self.has_multiple_ = 0 + self.multiple_ = 0 def has_multiple(self): return self.has_multiple_ @@ -967,6 +1006,10 @@ class Property(ProtocolBuffer.ProtocolMessage): if debug_strs is not None: debug_strs.append('Required field: value not set.') elif not self.value_.IsInitialized(debug_strs): initialized = 0 + if (not self.has_multiple_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: multiple not set.') return initialized def ByteSize(self): @@ -975,8 +1018,7 @@ class Property(ProtocolBuffer.ProtocolMessage): if (self.has_meaning_uri_): n += 1 + self.lengthString(len(self.meaning_uri_)) n += self.lengthString(len(self.name_)) n += self.lengthString(self.value_.ByteSize()) - if (self.has_multiple_): n += 2 - return n + 2 + return n + 4 def Clear(self): self.clear_meaning() @@ -994,9 +1036,8 @@ class Property(ProtocolBuffer.ProtocolMessage): out.putPrefixedString(self.meaning_uri_) out.putVarInt32(26) out.putPrefixedString(self.name_) - if (self.has_multiple_): - out.putVarInt32(32) - out.putBoolean(self.multiple_) + out.putVarInt32(32) + out.putBoolean(self.multiple_) out.putVarInt32(42) out.putVarInt32(self.value_.ByteSize()) self.value_.OutputUnchecked(out) @@ -1038,34 +1079,33 @@ class Property(ProtocolBuffer.ProtocolMessage): if self.has_multiple_: res+=prefix+("multiple: %s\n" % self.DebugFormatBool(self.multiple_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kmeaning = 1 kmeaning_uri = 2 kname = 3 kvalue = 5 kmultiple = 4 - _TEXT = ( - "ErrorCode", - "meaning", - "meaning_uri", - "name", - "multiple", - "value", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "meaning", + 2: "meaning_uri", + 3: "name", + 4: "multiple", + 5: "value", + }, 5) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.STRING, + }, 5, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1087,8 +1127,9 @@ class Path_Element(ProtocolBuffer.ProtocolMessage): self.type_ = x def clear_type(self): - self.has_type_ = 0 - self.type_ = "" + if self.has_type_: + self.has_type_ = 0 + self.type_ = "" def has_type(self): return self.has_type_ @@ -1099,8 +1140,9 @@ class Path_Element(ProtocolBuffer.ProtocolMessage): self.id_ = x def clear_id(self): - self.has_id_ = 0 - self.id_ = 0 + if self.has_id_: + self.has_id_ = 0 + self.id_ = 0 def has_id(self): return self.has_id_ @@ -1111,8 +1153,9 @@ class Path_Element(ProtocolBuffer.ProtocolMessage): self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -1264,30 +1307,30 @@ class Path(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kElementGroup = 1 kElementtype = 2 kElementid = 3 kElementname = 4 - _TEXT = ( - "ErrorCode", - "Element", - "type", - "id", - "name", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Element", + 2: "type", + 3: "id", + 4: "name", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STARTGROUP, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.STRING, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1307,8 +1350,9 @@ class Reference(ProtocolBuffer.ProtocolMessage): self.app_ = x def clear_app(self): - self.has_app_ = 0 - self.app_ = "" + if self.has_app_: + self.has_app_ = 0 + self.app_ = "" def has_app(self): return self.has_app_ @@ -1389,58 +1433,24 @@ class Reference(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res - kapp = 13 - kpath = 14 - - _TEXT = ( - "ErrorCode", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "app", - "path", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - ProtocolBuffer.Encoder.MAX_TYPE, + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.STRING, + kapp = 13 + kpath = 14 - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 13: "app", + 14: "path", + }, 14) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 13: ProtocolBuffer.Encoder.STRING, + 14: ProtocolBuffer.Encoder.STRING, + }, 14, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1453,6 +1463,8 @@ class User(ProtocolBuffer.ProtocolMessage): nickname_ = "" has_gaiaid_ = 0 gaiaid_ = 0 + has_obfuscated_gaiaid_ = 0 + obfuscated_gaiaid_ = "" def __init__(self, contents=None): if contents is not None: self.MergeFromString(contents) @@ -1464,8 +1476,9 @@ class User(ProtocolBuffer.ProtocolMessage): self.email_ = x def clear_email(self): - self.has_email_ = 0 - self.email_ = "" + if self.has_email_: + self.has_email_ = 0 + self.email_ = "" def has_email(self): return self.has_email_ @@ -1476,8 +1489,9 @@ class User(ProtocolBuffer.ProtocolMessage): self.auth_domain_ = x def clear_auth_domain(self): - self.has_auth_domain_ = 0 - self.auth_domain_ = "" + if self.has_auth_domain_: + self.has_auth_domain_ = 0 + self.auth_domain_ = "" def has_auth_domain(self): return self.has_auth_domain_ @@ -1488,8 +1502,9 @@ class User(ProtocolBuffer.ProtocolMessage): self.nickname_ = x def clear_nickname(self): - self.has_nickname_ = 0 - self.nickname_ = "" + if self.has_nickname_: + self.has_nickname_ = 0 + self.nickname_ = "" def has_nickname(self): return self.has_nickname_ @@ -1500,11 +1515,25 @@ class User(ProtocolBuffer.ProtocolMessage): self.gaiaid_ = x def clear_gaiaid(self): - self.has_gaiaid_ = 0 - self.gaiaid_ = 0 + if self.has_gaiaid_: + self.has_gaiaid_ = 0 + self.gaiaid_ = 0 def has_gaiaid(self): return self.has_gaiaid_ + def obfuscated_gaiaid(self): return self.obfuscated_gaiaid_ + + def set_obfuscated_gaiaid(self, x): + self.has_obfuscated_gaiaid_ = 1 + self.obfuscated_gaiaid_ = x + + def clear_obfuscated_gaiaid(self): + if self.has_obfuscated_gaiaid_: + self.has_obfuscated_gaiaid_ = 0 + self.obfuscated_gaiaid_ = "" + + def has_obfuscated_gaiaid(self): return self.has_obfuscated_gaiaid_ + def MergeFrom(self, x): assert x is not self @@ -1512,6 +1541,7 @@ class User(ProtocolBuffer.ProtocolMessage): if (x.has_auth_domain()): self.set_auth_domain(x.auth_domain()) if (x.has_nickname()): self.set_nickname(x.nickname()) if (x.has_gaiaid()): self.set_gaiaid(x.gaiaid()) + if (x.has_obfuscated_gaiaid()): self.set_obfuscated_gaiaid(x.obfuscated_gaiaid()) def Equals(self, x): if x is self: return 1 @@ -1523,6 +1553,8 @@ class User(ProtocolBuffer.ProtocolMessage): if self.has_nickname_ and self.nickname_ != x.nickname_: return 0 if self.has_gaiaid_ != x.has_gaiaid_: return 0 if self.has_gaiaid_ and self.gaiaid_ != x.gaiaid_: return 0 + if self.has_obfuscated_gaiaid_ != x.has_obfuscated_gaiaid_: return 0 + if self.has_obfuscated_gaiaid_ and self.obfuscated_gaiaid_ != x.obfuscated_gaiaid_: return 0 return 1 def IsInitialized(self, debug_strs=None): @@ -1547,6 +1579,7 @@ class User(ProtocolBuffer.ProtocolMessage): n += self.lengthString(len(self.auth_domain_)) if (self.has_nickname_): n += 1 + self.lengthString(len(self.nickname_)) n += self.lengthVarInt64(self.gaiaid_) + if (self.has_obfuscated_gaiaid_): n += 1 + self.lengthString(len(self.obfuscated_gaiaid_)) return n + 3 def Clear(self): @@ -1554,6 +1587,7 @@ class User(ProtocolBuffer.ProtocolMessage): self.clear_auth_domain() self.clear_nickname() self.clear_gaiaid() + self.clear_obfuscated_gaiaid() def OutputUnchecked(self, out): out.putVarInt32(10) @@ -1565,6 +1599,9 @@ class User(ProtocolBuffer.ProtocolMessage): out.putPrefixedString(self.nickname_) out.putVarInt32(32) out.putVarInt64(self.gaiaid_) + if (self.has_obfuscated_gaiaid_): + out.putVarInt32(42) + out.putPrefixedString(self.obfuscated_gaiaid_) def TryMerge(self, d): while d.avail() > 0: @@ -1581,6 +1618,9 @@ class User(ProtocolBuffer.ProtocolMessage): if tt == 32: self.set_gaiaid(d.getVarInt64()) continue + if tt == 42: + self.set_obfuscated_gaiaid(d.getPrefixedString()) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -1591,32 +1631,36 @@ class User(ProtocolBuffer.ProtocolMessage): if self.has_auth_domain_: res+=prefix+("auth_domain: %s\n" % self.DebugFormatString(self.auth_domain_)) if self.has_nickname_: res+=prefix+("nickname: %s\n" % self.DebugFormatString(self.nickname_)) if self.has_gaiaid_: res+=prefix+("gaiaid: %s\n" % self.DebugFormatInt64(self.gaiaid_)) + if self.has_obfuscated_gaiaid_: res+=prefix+("obfuscated_gaiaid: %s\n" % self.DebugFormatString(self.obfuscated_gaiaid_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kemail = 1 kauth_domain = 2 knickname = 3 kgaiaid = 4 - - _TEXT = ( - "ErrorCode", - "email", - "auth_domain", - "nickname", - "gaiaid", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + kobfuscated_gaiaid = 5 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "email", + 2: "auth_domain", + 3: "nickname", + 4: "gaiaid", + 5: "obfuscated_gaiaid", + }, 5) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.STRING, + }, 5, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -1680,8 +1724,9 @@ class EntityProto(ProtocolBuffer.ProtocolMessage): def mutable_owner(self): self.has_owner_ = 1; return self.owner() def clear_owner(self): - self.has_owner_ = 0; - if self.owner_ is not None: self.owner_.Clear() + if self.has_owner_: + self.has_owner_ = 0; + if self.owner_ is not None: self.owner_.Clear() def has_owner(self): return self.has_owner_ @@ -1692,8 +1737,9 @@ class EntityProto(ProtocolBuffer.ProtocolMessage): self.kind_ = x def clear_kind(self): - self.has_kind_ = 0 - self.kind_ = 0 + if self.has_kind_: + self.has_kind_ = 0 + self.kind_ = 0 def has_kind(self): return self.has_kind_ @@ -1704,8 +1750,9 @@ class EntityProto(ProtocolBuffer.ProtocolMessage): self.kind_uri_ = x def clear_kind_uri(self): - self.has_kind_uri_ = 0 - self.kind_uri_ = "" + if self.has_kind_uri_: + self.has_kind_uri_ = 0 + self.kind_uri_ = "" def has_kind_uri(self): return self.has_kind_uri_ @@ -1916,6 +1963,10 @@ class EntityProto(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kkey = 13 kentity_group = 16 kowner = 17 @@ -1924,64 +1975,27 @@ class EntityProto(ProtocolBuffer.ProtocolMessage): kproperty = 14 kraw_property = 15 - _TEXT = ( - "ErrorCode", - None, - None, - None, - "kind", - "kind_uri", - None, - None, - None, - None, - None, - None, - None, - "key", - "property", - "raw_property", - "entity_group", - "owner", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.MAX_TYPE, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 4: "kind", + 5: "kind_uri", + 13: "key", + 14: "property", + 15: "raw_property", + 16: "entity_group", + 17: "owner", + }, 17) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.STRING, + 13: ProtocolBuffer.Encoder.STRING, + 14: ProtocolBuffer.Encoder.STRING, + 15: ProtocolBuffer.Encoder.STRING, + 16: ProtocolBuffer.Encoder.STRING, + 17: ProtocolBuffer.Encoder.STRING, + }, 17, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -2000,8 +2014,9 @@ class CompositeProperty(ProtocolBuffer.ProtocolMessage): self.index_id_ = x def clear_index_id(self): - self.has_index_id_ = 0 - self.index_id_ = 0 + if self.has_index_id_: + self.has_index_id_ = 0 + self.index_id_ = 0 def has_index_id(self): return self.has_index_id_ @@ -2085,22 +2100,24 @@ class CompositeProperty(ProtocolBuffer.ProtocolMessage): cnt+=1 return res - kindex_id = 1 - kvalue = 2 - _TEXT = ( - "ErrorCode", - "index_id", - "value", - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.NUMERIC, + kindex_id = 1 + kvalue = 2 - ProtocolBuffer.Encoder.STRING, + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "index_id", + 2: "value", + }, 2) - ) + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -2132,8 +2149,9 @@ class Index_Property(ProtocolBuffer.ProtocolMessage): self.name_ = x def clear_name(self): - self.has_name_ = 0 - self.name_ = "" + if self.has_name_: + self.has_name_ = 0 + self.name_ = "" def has_name(self): return self.has_name_ @@ -2144,8 +2162,9 @@ class Index_Property(ProtocolBuffer.ProtocolMessage): self.direction_ = x def clear_direction(self): - self.has_direction_ = 0 - self.direction_ = 1 + if self.has_direction_: + self.has_direction_ = 0 + self.direction_ = 1 def has_direction(self): return self.has_direction_ @@ -2225,8 +2244,9 @@ class Index(ProtocolBuffer.ProtocolMessage): self.entity_type_ = x def clear_entity_type(self): - self.has_entity_type_ = 0 - self.entity_type_ = "" + if self.has_entity_type_: + self.has_entity_type_ = 0 + self.entity_type_ = "" def has_entity_type(self): return self.has_entity_type_ @@ -2237,8 +2257,9 @@ class Index(ProtocolBuffer.ProtocolMessage): self.ancestor_ = x def clear_ancestor(self): - self.has_ancestor_ = 0 - self.ancestor_ = 0 + if self.has_ancestor_: + self.has_ancestor_ = 0 + self.ancestor_ = 0 def has_ancestor(self): return self.has_ancestor_ @@ -2342,34 +2363,33 @@ class Index(ProtocolBuffer.ProtocolMessage): cnt+=1 return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kentity_type = 1 kancestor = 5 kPropertyGroup = 2 kPropertyname = 3 kPropertydirection = 4 - _TEXT = ( - "ErrorCode", - "entity_type", - "Property", - "name", - "direction", - "ancestor", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "entity_type", + 2: "Property", + 3: "name", + 4: "direction", + 5: "ancestor", + }, 5) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STARTGROUP, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + 5: ProtocolBuffer.Encoder.NUMERIC, + }, 5, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -2409,8 +2429,9 @@ class CompositeIndex(ProtocolBuffer.ProtocolMessage): self.app_id_ = x def clear_app_id(self): - self.has_app_id_ = 0 - self.app_id_ = "" + if self.has_app_id_: + self.has_app_id_ = 0 + self.app_id_ = "" def has_app_id(self): return self.has_app_id_ @@ -2421,8 +2442,9 @@ class CompositeIndex(ProtocolBuffer.ProtocolMessage): self.id_ = x def clear_id(self): - self.has_id_ = 0 - self.id_ = 0 + if self.has_id_: + self.has_id_ = 0 + self.id_ = 0 def has_id(self): return self.has_id_ @@ -2441,8 +2463,9 @@ class CompositeIndex(ProtocolBuffer.ProtocolMessage): self.state_ = x def clear_state(self): - self.has_state_ = 0 - self.state_ = 0 + if self.has_state_: + self.has_state_ = 0 + self.state_ = 0 def has_state(self): return self.has_state_ @@ -2545,30 +2568,30 @@ class CompositeIndex(ProtocolBuffer.ProtocolMessage): if self.has_state_: res+=prefix+("state: %s\n" % self.DebugFormatInt32(self.state_)) return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kapp_id = 1 kid = 2 kdefinition = 3 kstate = 4 - _TEXT = ( - "ErrorCode", - "app_id", - "id", - "definition", - "state", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.NUMERIC, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "app_id", + 2: "id", + 3: "definition", + 4: "state", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.NUMERIC, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.NUMERIC, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" google_appengine/google/appengine/datastore/entity_pb.py @@ -41,11 +41,21 @@ import urllib import urlparse import wsgiref.handlers +try: + from google.appengine.cron import groctimespecification + from google.appengine.api import croninfo +except ImportError: + HAVE_CRON = False +else: + HAVE_CRON = True + +from google.appengine.api import apiproxy_stub_map from google.appengine.api import datastore from google.appengine.api import datastore_admin from google.appengine.api import datastore_types from google.appengine.api import datastore_errors from google.appengine.api import memcache +from google.appengine.api.labs import taskqueue from google.appengine.api import users from google.appengine.ext import db from google.appengine.ext import webapp @@ -108,7 +118,13 @@ class BaseRequestHandler(webapp.RequestHandler): 'interactive_path': base_path + InteractivePageHandler.PATH, 'interactive_execute_path': base_path + InteractiveExecuteHandler.PATH, 'memcache_path': base_path + MemcachePageHandler.PATH, + 'queues_path': base_path + QueuesPageHandler.PATH, + 'xmpp_path': base_path + XMPPPageHandler.PATH, + 'inboundmail_path': base_path + InboundMailPageHandler.PATH, } + if HAVE_CRON: + values['cron_path'] = base_path + CronPageHandler.PATH + values.update(template_values) directory = os.path.dirname(__file__) path = os.path.join(directory, os.path.join('templates', template_name)) @@ -201,6 +217,141 @@ class InteractiveExecuteHandler(BaseRequestHandler): self.generate('interactive-output.html', {'output': results}) +class CronPageHandler(BaseRequestHandler): + """Shows information about configured cron jobs in this application.""" + PATH = '/cron' + + def get(self, now=None): + """Shows template displaying the configured cron jobs.""" + if not now: + now = datetime.datetime.now() + values = {'request': self.request} + cron_info = _ParseCronYaml() + values['cronjobs'] = [] + values['now'] = str(now) + if cron_info and cron_info.cron: + for entry in cron_info.cron: + job = {} + values['cronjobs'].append(job) + if entry.description: + job['description'] = entry.description + else: + job['description'] = '(no description)' + if entry.timezone: + job['timezone'] = entry.timezone + job['url'] = entry.url + job['schedule'] = entry.schedule + schedule = groctimespecification.GrocTimeSpecification(entry.schedule) + matches = schedule.GetMatches(now, 3) + job['times'] = [] + for match in matches: + job['times'].append({'runtime': match.strftime("%Y-%m-%d %H:%M:%SZ"), + 'difference': str(match - now)}) + self.generate('cron.html', values) + + +class XMPPPageHandler(BaseRequestHandler): + """Tests XMPP requests.""" + PATH = '/xmpp' + + def get(self): + """Shows template displaying the XMPP.""" + xmpp_configured = True + values = { + 'xmpp_configured': xmpp_configured, + 'request': self.request + } + self.generate('xmpp.html', values) + + +class InboundMailPageHandler(BaseRequestHandler): + """Tests Mail requests.""" + PATH = '/inboundmail' + + def get(self): + """Shows template displaying the Inbound Mail form.""" + inboundmail_configured = True + values = { + 'inboundmail_configured': inboundmail_configured, + 'request': self.request + } + self.generate('inboundmail.html', values) + + +class QueuesPageHandler(BaseRequestHandler): + """Shows information about configured (and default) task queues.""" + PATH = '/queues' + + def __init__(self): + self.stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') + + def get(self): + """Shows template displaying the configured task queues.""" + values = { + 'request': self.request, + 'queues': self.stub.GetQueues(), + } + self.generate('queues.html', values) + + def post(self): + """Handle modifying actions and/or redirect to GET page.""" + + if self.request.get('action:flushqueue'): + self.stub.FlushQueue(self.request.get('queue')) + self.redirect(self.request.path_url) + + +class TasksPageHandler(BaseRequestHandler): + """Shows information about a queue's tasks.""" + + PATH = '/tasks' + + PAGE_SIZE = 20 + + def __init__(self): + self.stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') + + def get(self): + """Shows template displaying the queue's tasks.""" + queue = self.request.get('queue') + start = int(self.request.get('start', 0)) + all_tasks = self.stub.GetTasks(queue) + + next_start = start + self.PAGE_SIZE + tasks = all_tasks[start:next_start] + current_page = int(start / self.PAGE_SIZE) + 1 + pages = [] + for number in xrange(int(math.ceil(len(all_tasks) / + float(self.PAGE_SIZE)))): + pages.append({ + 'number': number + 1, + 'start': number * self.PAGE_SIZE + }) + if not all_tasks[next_start:]: + next_start = -1 + prev_start = start - self.PAGE_SIZE + if prev_start < 0: + prev_start = -1 + + values = { + 'request': self.request, + 'queue_name': queue, + 'tasks': tasks, + 'start_base_url': self.filter_url(['queue']), + 'prev_start': prev_start, + 'next_start': next_start, + 'pages': pages, + 'current_page': current_page, + } + self.generate('tasks.html', values) + + def post(self): + if self.request.get('action:deletetask'): + self.stub.DeleteTask(self.request.get('queue'), self.request.get('task')) + self.redirect(self.request.path_url + '?queue=' + self.request.get('queue')) + return + + class MemcachePageHandler(BaseRequestHandler): """Shows stats about memcache and query form to get values.""" PATH = '/memcache' @@ -1057,6 +1208,9 @@ class NoneType(DataType): def parse(self, value): return None + def python_type(self): + return None + def format(self, value): return 'None' @@ -1089,8 +1243,29 @@ for data_type in _DATA_TYPES.values(): _NAMED_DATA_TYPES[data_type.name()] = data_type +def _ParseCronYaml(): + """Loads the cron.yaml file and parses it. + + The CWD of the dev_appserver is the root of the application here. + + Returns a dict representing the contents of cron.yaml. + """ + cronyaml_files = 'cron.yaml', 'cron.yml' + for cronyaml in cronyaml_files: + try: + fh = open(cronyaml, "r") + except IOError: + continue + try: + cron_info = croninfo.LoadSingleCron(fh) + return cron_info + finally: + fh.close() + return None + + def main(): - application = webapp.WSGIApplication([ + handlers = [ ('.*' + DatastoreQueryHandler.PATH, DatastoreQueryHandler), ('.*' + DatastoreEditHandler.PATH, DatastoreEditHandler), ('.*' + DatastoreBatchEditHandler.PATH, DatastoreBatchEditHandler), @@ -1098,8 +1273,15 @@ def main(): ('.*' + InteractiveExecuteHandler.PATH, InteractiveExecuteHandler), ('.*' + MemcachePageHandler.PATH, MemcachePageHandler), ('.*' + ImageHandler.PATH, ImageHandler), + ('.*' + QueuesPageHandler.PATH, QueuesPageHandler), + ('.*' + TasksPageHandler.PATH, TasksPageHandler), + ('.*' + XMPPPageHandler.PATH, XMPPPageHandler), + ('.*' + InboundMailPageHandler.PATH, InboundMailPageHandler), ('.*', DefaultPageHandler), - ], debug=_DEBUG) + ] + if HAVE_CRON: + handlers.insert(0, ('.*' + CronPageHandler.PATH, CronPageHandler)) + application = webapp.WSGIApplication(handlers, debug=_DEBUG) wsgiref.handlers.CGIHandler().run(application) google_appengine/google/appengine/ext/admin/__init__.py @@ -10,48 +10,54 @@ </head> <body {% block bodyattributes %}{% endblock %}> <div class="g-doc"> - + <div id="hd" class="g-section"> <div class="g-section"> <img id="ae-logo" src="./images/google.gif" width="153" height="47" alt="Google App Engine"/> </div> - + <div id="ae-appbar-lrg" class="g-section"> <h1>{{ application_name }} Development Console</h1> </div> - + </div> - - + + <div id="bd" class="g-section"> - + <div class="g-section g-tpl-160"> - + <div id="ae-lhs-nav" class="g-unit g-first"> - + <div id="ae-nav" class="g-c"> - + <ul id="menu"> <li><a href="{{ datastore_path }}">Datastore Viewer</a></li> <li><a href="{{ interactive_path }}">Interactive Console</a></li> <li><a href="{{ memcache_path }}">Memcache Viewer</a></li> + <li><a href="{{ queues_path }}">Task Queues</a></li> + {% if cron_path %} + <li><a href="{{ cron_path }}">Cron Jobs</a></li> + {% endif %} + <li><a href="{{ xmpp_path }}">XMPP</a></li> + <li><a href="{{ inboundmail_path }}">Inbound Mail</a></li> </ul> - + </div> - + </div> - + <div id="ae-content" class="g-unit"> {% block body %}{% endblock %} </div> - + </div> - + <div id="ft"> <p> - &copy;2008 Google + &copy;2009 Google </p> </div> {% block final %}{% endblock %} @@ -79,7 +85,7 @@ function makeSelected(e) { e.className = "ae-nav-selected"; } - + walk(document.getElementById("menu"), isCurrentLink, makeSelected); //]]> google_appengine/google/appengine/ext/admin/templates/base.html @@ -42,4 +42,129 @@ h1 { #ae-content { padding-left: 1em; border-left: 3px solid #e5ecf9; + min-height: 200px; +} + +/* Tables */ +.ae-table-plain { + border-collapse: collapse; + width: 100%; +} +.ae-table { + border: 1px solid #c5d7ef; + border-collapse: collapse; + width: 100%; +} + +#bd h2.ae-table-title { + background: #e5ecf9; + margin: 0; + color: #000; + font-size: 1em; + padding: 3px 0 3px 5px; + border-left: 1px solid #c5d7ef; + border-right: 1px solid #c5d7ef; + border-top: 1px solid #c5d7ef; +} +.ae-table-caption, +.ae-table caption { + border: 1px solid #c5d7ef; + background: #e5ecf9; + /** + * Fixes the caption margin ff display bug. + * see www.aurora-il.org/table_test.htm + * this is a slight variation to specifically target FF since Safari + * was shifting the caption over in an ugly fashion with margin-left: -1px + */ + -moz-margin-start: -1px; +} +.ae-table caption { + padding: 3px 5px; + text-align: left; +} +.ae-table th, +.ae-table td { + background-color: #fff; + padding: .35em 1em .25em .35em; + margin: 0; +} +.ae-table thead th { + font-weight: bold; + text-align: left; + background: #c5d7ef; + vertical-align: bottom; +} +.ae-table tfoot tr td { + border-top: 1px solid #c5d7ef; + background-color: #e5ecf9; +} +.ae-table td { + border-top: 1px solid #c5d7ef; + border-bottom: 1px solid #c5d7ef; +} +.ae-even td, +.ae-even th, +.ae-even-top td, +.ae-even-tween td, +.ae-even-bottom td, +ol.ae-even { + background-color: #e9e9e9; + border-top: 1px solid #c5d7ef; + border-bottom: 1px solid #c5d7ef; +} +.ae-even-top td { + border-bottom: 0; +} +.ae-even-bottom td { + border-top: 0; +} +.ae-even-tween td { + border: 0; +} +.ae-table .ae-tween td { + border: 0; +} +.ae-table .ae-tween-top td { + border-bottom: 0; +} +.ae-table .ae-tween-bottom td { + border-top: 0; +} +.ae-table #ae-live td { + background-color: #ffeac0; +} +.ae-table-fixed { + table-layout: fixed; +} +.ae-table-fixed td, +.ae-table-nowrap { + overflow: hidden; + white-space: nowrap; +} +.ae-new-usr td { + border-top: 1px solid #ccccce; + background-color: #ffe; +} +.ae-error-td td { + border: 2px solid #f00; + background-color: #fee; +} +.ae-table .ae-pager { + background-color: #c5d7ef; +} + +.ae-errorbox { + border: 1px solid #f00; + background-color: #fee; + margin-bottom: 1em; + padding: 1em; + display: inline-block; +} + +.ae-message { + border: 1px solid #e5ecf9; + background-color: #f6f9ff; + margin-bottom: 1em; + padding: 1em; + display: inline-block; } google_appengine/google/appengine/ext/admin/templates/css/ae.css @@ -105,17 +105,12 @@ import StringIO import csv import httplib import os -import sys import traceback -import types -import struct -import zlib import google import wsgiref.handlers from google.appengine.api import datastore -from google.appengine.api import datastore_types from google.appengine.ext import webapp from google.appengine.ext.bulkload import constants @@ -299,13 +294,8 @@ class BulkLoad(webapp.RequestHandler): """ Handle a POST. Reads CSV data, converts to entities, and stores them. """ self.response.headers['Content-Type'] = 'text/plain' - version = self.request.headers.get('GAE-Uploader-Version', '0') - if version == '1': - kind = self.request.headers.get('GAE-Uploader-Kind') - response, output = self.LoadV1(kind, self.request.body) - else: - response, output = self.Load(self.request.get(constants.KIND_PARAM), - self.request.get(constants.CSV_PARAM)) + response, output = self.Load(self.request.get(constants.KIND_PARAM), + self.request.get(constants.CSV_PARAM)) self.response.set_status(response) self.response.out.write(output) @@ -422,69 +412,6 @@ class BulkLoad(webapp.RequestHandler): return self.LoadEntities(self.IterRows(reader), loader) - def IterRowsV1(self, data): - """Yields a tuple of columns for each row in the uploaded data. - - Args: - data: a string containing the unzipped v1 format data to load. - - """ - column_count, = struct.unpack_from('!i', data) - offset = 4 - - lengths_format = '!%di' % (column_count,) - - while offset < len(data): - id_num = struct.unpack_from('!i', data, offset=offset) - offset += 4 - - value_lengths = struct.unpack_from(lengths_format, data, offset=offset) - offset += 4 * column_count - - columns = struct.unpack_from(''.join('%ds' % length - for length in value_lengths), data, - offset=offset) - offset += sum(value_lengths) - - yield (id_num, columns) - - - def LoadV1(self, kind, data): - """Parses version-1 format data, converts to entities, and stores them. - - On error, fails fast. Returns a "bad request" HTTP response code and - includes the traceback in the output. - - Args: - kind: a string containing the entity kind that this loader handles - data: a string containing the (v1 format) data to load - - Returns: - tuple (response code, output) where: - response code: integer HTTP response code to return - output: string containing the HTTP response body - """ - Validate(kind, basestring) - Validate(data, basestring) - output = [] - - try: - loader = Loader.RegisteredLoaders()[kind] - except KeyError: - output.append('Error: no Loader defined for kind %s.' % kind) - return (httplib.BAD_REQUEST, ''.join(output)) - - try: - data = zlib.decompress(data) - except: - stacktrace = traceback.format_exc() - output.append('Error: Could not decompress data\n%s' % stacktrace) - return (httplib.BAD_REQUEST, ''.join(output)) - - key_format = 'i%010d' - return self.LoadEntities(self.IterRowsV1(data), - loader, - key_format=key_format) def main(*loaders): """Starts bulk upload. google_appengine/google/appengine/ext/bulkload/__init__.py @@ -77,15 +77,21 @@ preconfigured to return all matching comments: + +import base64 +import copy import datetime import logging +import re import time import urlparse +import warnings from google.appengine.api import datastore from google.appengine.api import datastore_errors from google.appengine.api import datastore_types from google.appengine.api import users +from google.appengine.datastore import datastore_pb Error = datastore_errors.Error BadValueError = datastore_errors.BadValueError @@ -118,6 +124,10 @@ Rating = datastore_types.Rating Text = datastore_types.Text Blob = datastore_types.Blob ByteString = datastore_types.ByteString +BlobKey = datastore_types.BlobKey + +READ_CAPABILITY = datastore.READ_CAPABILITY +WRITE_CAPABILITY = datastore.WRITE_CAPABILITY _kind_map = {} @@ -182,11 +192,17 @@ _ALLOWED_PROPERTY_TYPES = set([ PhoneNumber, PostalAddress, Rating, + BlobKey, ]) _ALLOWED_EXPANDO_PROPERTY_TYPES = set(_ALLOWED_PROPERTY_TYPES) _ALLOWED_EXPANDO_PROPERTY_TYPES.update((list, tuple, type(None))) +_OPERATORS = ['<', '<=', '>', '>=', '=', '==', '!=', 'in'] +_FILTER_REGEX = re.compile( + '^\s*([^\s]+)(\s+(%s)\s*)?$' % '|'.join(_OPERATORS), + re.IGNORECASE | re.UNICODE) + def class_for_kind(kind): """Return base-class responsible for implementing kind. @@ -232,6 +248,49 @@ def check_reserved_word(attr_name): "definition." % locals()) +def query_descendants(model_instance): + """Returns a query for all the descendants of a model instance. + + Args: + model_instance: Model instance to find the descendants of. + + Returns: + Query that will retrieve all entities that have the given model instance + as an ancestor. Unlike normal ancestor queries, this does not include the + ancestor itself. + """ + + result = Query().ancestor(model_instance); + result.filter(datastore_types._KEY_SPECIAL_PROPERTY + ' >', + model_instance.key()); + return result; + + +def model_to_protobuf(model_instance, _entity_class=datastore.Entity): + """Encodes a model instance as a protocol buffer. + + Args: + model_instance: Model instance to encode. + Returns: + entity_pb.EntityProto representation of the model instance + """ + return model_instance._populate_entity(_entity_class).ToPb() + + +def model_from_protobuf(pb, _entity_class=datastore.Entity): + """Decodes a model instance from a protocol buffer. + + Args: + pb: The protocol buffer representation of the model instance. Can be an + entity_pb.EntityProto or str encoding of an entity_bp.EntityProto + + Returns: + Model instance resulting from decoding the protocol buffer + """ + entity = _entity_class.FromPb(pb) + return class_for_kind(entity.kind()).from_entity(entity) + + def _initialize_properties(model_class, name, bases, dct): """Initialize Property attributes for Model-class. @@ -239,17 +298,31 @@ def _initialize_properties(model_class, name, bases, dct): model_class: Model class to initialize properties for. """ model_class._properties = {} + property_source = {} + + def get_attr_source(name, cls): + for src_cls in cls.mro(): + if name in src_cls.__dict__: + return src_cls + defined = set() for base in bases: if hasattr(base, '_properties'): - property_keys = base._properties.keys() - duplicate_properties = defined.intersection(property_keys) - if duplicate_properties: - raise DuplicatePropertyError( - 'Duplicate properties in base class %s already defined: %s' % - (base.__name__, list(duplicate_properties))) - defined.update(property_keys) - model_class._properties.update(base._properties) + property_keys = set(base._properties.keys()) + duplicate_property_keys = defined & property_keys + for dupe_prop_name in duplicate_property_keys: + old_source = property_source[dupe_prop_name] = get_attr_source( + dupe_prop_name, property_source[dupe_prop_name]) + new_source = get_attr_source(dupe_prop_name, base) + if old_source != new_source: + raise DuplicatePropertyError( + 'Duplicate property, %s, is inherited from both %s and %s.' % + (dupe_prop_name, old_source.__name__, new_source.__name__)) + property_keys -= duplicate_property_keys + if property_keys: + defined |= property_keys + property_source.update(dict.fromkeys(property_keys, base)) + model_class._properties.update(base._properties) for attr_name in dct.keys(): attr = dct[attr_name] @@ -261,6 +334,9 @@ def _initialize_properties(model_class, name, bases, dct): model_class._properties[attr_name] = attr attr.__property_config__(model_class, attr_name) + model_class._unindexed_properties = frozenset( + name for name, prop in model_class._properties.items() if not prop.indexed) + class PropertiedClass(type): """Meta-class for initializing Model classes properties. @@ -328,8 +404,14 @@ class Property(object): creation_counter = 0 - def __init__(self, verbose_name=None, name=None, default=None, - required=False, validator=None, choices=None): + def __init__(self, + verbose_name=None, + name=None, + default=None, + required=False, + validator=None, + choices=None, + indexed=True): """Initializes this Property with the given options. Args: @@ -340,6 +422,7 @@ class Property(object): required: Whether property is required. validator: User provided method used for validation. choices: User provided set of valid property values. + indexed: Whether property is indexed. """ self.verbose_name = verbose_name self.name = name @@ -347,6 +430,7 @@ class Property(object): self.required = required self.validator = validator self.choices = choices + self.indexed = indexed self.creation_counter = Property.creation_counter Property.creation_counter += 1 @@ -481,6 +565,21 @@ class Property(object): """ return value + def _require_parameter(self, kwds, parameter, value): + """Sets kwds[parameter] to value. + + If kwds[parameter] exists and is not value, raises ConfigurationError. + + Args: + kwds: The parameter dict, which maps parameter names (strings) to values. + parameter: The name of the parameter to set. + value: The value to set it to. + """ + if parameter in kwds and kwds[parameter] != value: + raise ConfigurationError('%s must be %s.' % (parameter, value)) + + kwds[parameter] = value + def _attr_name(self): """Attribute name we use for this property in model instances. @@ -522,17 +621,18 @@ class Model(object): def __init__(self, parent=None, key_name=None, + key=None, _app=None, _from_entity=False, **kwds): """Creates a new instance of this model. - To create a new entity, you instantiate a model and then call save(), + To create a new entity, you instantiate a model and then call put(), which saves the entity to the datastore: person = Person() person.name = 'Bret' - person.save() + person.put() You can initialize properties in the model in the constructor with keyword arguments: @@ -547,38 +647,64 @@ class Model(object): parent: Parent instance for this instance or None, indicating a top- level instance. key_name: Name for new model instance. - _app: Intentionally undocumented. + key: Key instance for this instance, overrides parent and key_name _from_entity: Intentionally undocumented. args: Keyword arguments mapping to properties of model. """ - if key_name == '': - raise BadKeyError('Name cannot be empty.') - elif key_name is not None and not isinstance(key_name, basestring): - raise BadKeyError('Name must be string type, not %s' % - key_name.__class__.__name__) - - if parent is not None: - if not isinstance(parent, (Model, Key)): - raise TypeError('Expected Model type; received %s (is %s)' % - (parent, parent.__class__.__name__)) - if isinstance(parent, Model) and not parent.has_key(): - raise BadValueError( - "%s instance must have a complete key before it can be used as a " - "parent." % parent.kind()) - if isinstance(parent, Key): - self._parent_key = parent - self._parent = None - else: - self._parent_key = parent.key() - self._parent = parent - else: - self._parent_key = None + if key is not None: + if isinstance(key, (tuple, list)): + key = Key.from_path(*key) + if isinstance(key, basestring): + key = Key(encoded=key) + if not isinstance(key, Key): + raise TypeError('Expected Key type; received %s (is %s)' % + (key, key.__class__.__name__)) + if not key.has_id_or_name(): + raise BadKeyError('Key must have an id or name') + if key.kind() != self.kind(): + raise BadKeyError('Expected Key kind to be %s; received %s' % + (self.kind(), key.kind())) + if _app is not None and key.app() != _app: + raise BadKeyError('Expected Key app to be %s; received %s' % + (_app, key.app())) + if key_name is not None: + raise BadArgumentError('Cannot use key and key_name at the same time') + if parent is not None: + raise BadArgumentError('Cannot use key and parent at the same time') + self._key = key + self._key_name = None self._parent = None + self._parent_key = None + else: + if key_name == '': + raise BadKeyError('Name cannot be empty.') + elif key_name is not None and not isinstance(key_name, basestring): + raise BadKeyError('Name must be string type, not %s' % + key_name.__class__.__name__) + + if parent is not None: + if not isinstance(parent, (Model, Key)): + raise TypeError('Expected Model type; received %s (is %s)' % + (parent, parent.__class__.__name__)) + if isinstance(parent, Model) and not parent.has_key(): + raise BadValueError( + "%s instance must have a complete key before it can be used as a " + "parent." % parent.kind()) + if isinstance(parent, Key): + self._parent_key = parent + self._parent = None + else: + self._parent_key = parent.key() + self._parent = parent + else: + self._parent_key = None + self._parent = None + self._key_name = key_name + self._key = None + self._entity = None - self._key_name = key_name self._app = _app - properties = self.properties() for prop in self.properties().values(): if prop.name in kwds: value = kwds[prop.name] @@ -594,8 +720,9 @@ class Model(object): """Unique key for this entity. This property is only available if this entity is already stored in the - datastore, so it is available if this entity was fetched returned from a - query, or after save() is called the first time for new entities. + datastore or if it has a full key, so it is available if this entity was + fetched returned from a query, or after put() is called the first time + for new entities, or if a complete key was given when constructed. Returns: Datastore key of persisted entity. @@ -605,9 +732,12 @@ class Model(object): """ if self.is_saved(): return self._entity.key() + elif self._key: + return self._key elif self._key_name: - parent = self._parent and self._parent.key() - return Key.from_path(self.kind(), self._key_name, parent=parent) + parent = self._parent_key or (self._parent and self._parent.key()) + self._key = Key.from_path(self.kind(), self._key_name, parent=parent) + return self._key else: raise NotSavedError() @@ -636,8 +766,11 @@ class Model(object): Populated self._entity """ self._entity = self._populate_entity(_entity_class=_entity_class) - if hasattr(self, '_key_name'): - del self._key_name + for attr in ('_key_name', '_key'): + try: + delattr(self, attr) + except AttributeError: + pass return self._entity def put(self): @@ -673,20 +806,23 @@ class Model(object): if self.is_saved(): entity = self._entity else: - if self._parent_key is not None: - entity = _entity_class(self.kind(), - parent=self._parent_key, - name=self._key_name, - _app=self._app) - elif self._parent is not None: - entity = _entity_class(self.kind(), - parent=self._parent._entity, - name=self._key_name, - _app=self._app) + kwds = {'_app': self._app, + 'unindexed_properties': self._unindexed_properties} + if self._key is not None: + if self._key.id(): + kwds['id'] = self._key.id() + else: + kwds['name'] = self._key.name() + if self._key.parent(): + kwds['parent'] = self._key.parent() else: - entity = _entity_class(self.kind(), - name=self._key_name, - _app=self._app) + if self._key_name is not None: + kwds['name'] = self._key_name + if self._parent_key is not None: + kwds['parent'] = self._parent_key + elif self._parent is not None: + kwds['parent'] = self._parent._entity + entity = _entity_class(self.kind(), **kwds) self._to_entity(entity) return entity @@ -715,14 +851,15 @@ class Model(object): def has_key(self): """Determine if this model instance has a complete key. - Ids are not assigned until the data is saved to the Datastore, but - instances with a key name always have a full key. + When not using a fully self-assigned Key, ids are not assigned until the + data is saved to the Datastore, but instances with a key name always have + a full key. Returns: - True if the object has been persisted to the datastore or has a key_name, - otherwise False. + True if the object has been persisted to the datastore or has a key + or has a key_name, otherwise False. """ - return self.is_saved() or self._key_name + return self.is_saved() or self._key or self._key_name def dynamic_properties(self): """Returns a list of all dynamic properties defined for instance.""" @@ -760,6 +897,8 @@ class Model(object): return self._parent.key() elif self._entity is not None: return self._entity.parent() + elif self._key is not None: + return self._key.parent() else: return None @@ -920,13 +1059,13 @@ class Model(object): return run_in_transaction(txn) @classmethod - def all(cls): + def all(cls, **kwds): """Returns a query over all instances of this model from the datastore. Returns: Query that will retrieve all instances from entity collection. """ - return Query(cls) + return Query(cls, **kwds) @classmethod def gql(cls, query_string, *args, **kwds): @@ -983,8 +1122,12 @@ class Model(object): entity_values = cls._load_entity_values(entity) instance = cls(None, _from_entity=True, **entity_values) - instance._entity = entity - del instance._key_name + if entity.is_saved(): + instance._entity = entity + del instance._key_name + del instance._key + elif entity.key().has_id_or_name(): + instance._key = entity.key() return instance @classmethod @@ -1092,6 +1235,33 @@ def delete(models): keys.append(key) datastore.Delete(keys) +def allocate_ids(model, size): + """Allocates a range of IDs of size for the model_key defined by model + + Allocates a range of IDs in the datastore such that those IDs will not + be automatically assigned to new entities. You can only allocate IDs + for model keys from your app. If there is an error, raises a subclass of + datastore_errors.Error. + + Args: + model: Model, Key or string to serve as a model specifying the ID sequence + in which to allocate IDs + + Returns: + (start, end) of the allocated range, inclusive. + """ + models_or_keys, multiple = datastore.NormalizeAndTypeCheck( + model, (Model, Key, basestring)) + keys = [] + for model_or_key in models_or_keys: + if isinstance(model_or_key, Model): + key = model_or_key = model_or_key.key() + elif isinstance(model_or_key, basestring): + key = model_or_key = Key(model_or_key) + else: + key = model_or_key + keys.append(key) + return datastore.AllocateIds(keys, size) class Expando(Model): """Dynamically expandable model. @@ -1143,7 +1313,7 @@ class Expando(Model): 1 - Because it is not possible for the datastore to know what kind of property to store on an undefined expando value, setting a property to - None is the same as deleting it form the expando. + None is the same as deleting it from the expando. 2 - Persistent variables on Expando must not begin with '_'. These variables considered to be 'protected' in Python, and are used @@ -1171,7 +1341,7 @@ class Expando(Model): super(Expando, self).__init__(parent, key_name, _app, **kwds) self._dynamic_properties = {} for prop, value in kwds.iteritems(): - if prop not in self.properties() and value is not None: + if prop not in self.properties(): setattr(self, prop, value) def __setattr__(self, key, value): @@ -1287,14 +1457,29 @@ class Expando(Model): class _BaseQuery(object): """Base class for both Query and GqlQuery.""" + _compile = False + def __init__(self, model_class=None, keys_only=False, compile=True, + cursor=None): + """Constructor. - def __init__(self, model_class): - """Constructor." - - Args: - model_class: Model class from which entities are constructed. + Args: + model_class: Model class from which entities are constructed. + keys_only: Whether the query should return full entities or only keys. + compile: Whether the query should also return a compiled query. + cursor: A compiled query from which to resume. """ self._model_class = model_class + self._keys_only = keys_only + self._compile = compile + self._cursor = cursor + + def is_keys_only(self): + """Returns whether this query is keys only. + + Returns: + True if this query returns keys, False if it returns entities. + """ + return self._keys_only def _get_query(self): """Subclass must override (and not call their super method). @@ -1313,7 +1498,14 @@ class _BaseQuery(object): Returns: Iterator for this query. """ - return _QueryIterator(self._model_class, iter(self._get_query().Run())) + self._compile = False + raw_query = self._get_query() + iterator = raw_query.Run() + + if self._keys_only: + return iterator + else: + return _QueryIterator(self._model_class, iter(iterator)) def __iter__(self): """Iterator for this query. @@ -1350,7 +1542,10 @@ class _BaseQuery(object): Returns: Number of entities this query fetches. """ - return self._get_query().Count(limit=limit) + self._compile = False + raw_query = self._get_query() + result = raw_query.Count(limit=limit) + return result def fetch(self, limit, offset=0): """Return a list of items selected using SQL-like limit and offset. @@ -1375,8 +1570,51 @@ class _BaseQuery(object): raise ValueError('Arguments to fetch() must be >= 0') if limit == 0: return [] - raw = self._get_query().Get(limit, offset) - return map(self._model_class.from_entity, raw) + raw_query = self._get_query() + raw = raw_query.Get(limit, offset) + if self._compile: + try: + self._compiled_query = raw_query.GetCompiledQuery() + except AssertionError, e: + self._compiled_query = e + + if self._keys_only: + return raw + else: + if self._model_class is not None: + return [self._model_class.from_entity(e) for e in raw] + else: + return [class_for_kind(e.kind()).from_entity(e) for e in raw] + + def cursor(self): + if not self._compile: + raise AssertionError('No cursor available, this action does not support ' + 'cursors (try "fetch" instead)') + try: + if not self._compiled_query: + return self._compiled_query + if isinstance(self._compiled_query, Exception): + raise self._compiled_query + return base64.urlsafe_b64encode(self._compiled_query.Encode()) + except AttributeError: + raise AssertionError('No cursor available, this query has not been ' + 'executed') + + def with_cursor(self, cursor): + try: + assert cursor, "Cursor cannot be empty" + cursor = datastore_pb.CompiledQuery(base64.urlsafe_b64decode(cursor)) + assert cursor.IsInitialized() + except (AssertionError, TypeError), e: + raise datastore_errors.BadValueError( + 'Invalid cursor %s. Details: %s' % (cursor, e)) + except Exception, e: + if e.__class__.__name__ == 'ProtocolBufferDecodeError': + raise datastore_errors.BadValueError('Invalid cursor %s.' % cursor) + else: + raise + self._cursor = cursor + return self def __getitem__(self, arg): """Support for query[index] and query[start:stop]. @@ -1453,7 +1691,11 @@ class _QueryIterator(object): Raises: StopIteration when there are no more results in query. """ - return self.__model_class.from_entity(self.__iterator.next()) + if self.__model_class is not None: + return self.__model_class.from_entity(self.__iterator.next()) + else: + entity = self.__iterator.next() + return class_for_kind(entity.kind()).from_entity(entity) def _normalize_query_parameter(value): @@ -1517,23 +1759,88 @@ class Query(_BaseQuery): print story.title """ - def __init__(self, model_class): + def __init__(self, model_class=None, keys_only=False, cursor=None): """Constructs a query over instances of the given Model. Args: model_class: Model class to build query for. + keys_only: Whether the query should return full entities or only keys. + cursor: A compiled query from which to resume. """ - super(Query, self).__init__(model_class) - self.__query_set = {} + super(Query, self).__init__(model_class, keys_only, cursor=cursor) + self.__query_sets = [{}] self.__orderings = [] self.__ancestor = None - def _get_query(self, _query_class=datastore.Query): - query = _query_class(self._model_class.kind(), self.__query_set) - if self.__ancestor is not None: - query.Ancestor(self.__ancestor) - query.Order(*self.__orderings) - return query + def _get_query(self, + _query_class=datastore.Query, + _multi_query_class=datastore.MultiQuery): + queries = [] + for query_set in self.__query_sets: + if self._model_class is not None: + kind = self._model_class.kind() + else: + kind = None + query = _query_class(kind, + query_set, + keys_only=self._keys_only, + compile=self._compile, + cursor=self._cursor) + query.Order(*self.__orderings) + if self.__ancestor is not None: + query.Ancestor(self.__ancestor) + queries.append(query) + + if (_query_class != datastore.Query and + _multi_query_class == datastore.MultiQuery): + warnings.warn( + 'Custom _query_class specified without corresponding custom' + ' _query_multi_class. Things will break if you use queries with' + ' the "IN" or "!=" operators.', RuntimeWarning) + if len(queries) > 1: + raise datastore_errors.BadArgumentError( + 'Query requires multiple subqueries to satisfy. If _query_class' + ' is overridden, _multi_query_class must also be overridden.') + elif (_query_class == datastore.Query and + _multi_query_class != datastore.MultiQuery): + raise BadArgumentError('_query_class must also be overridden if' + ' _multi_query_class is overridden.') + + if len(queries) == 1: + return queries[0] + else: + return _multi_query_class(queries, self.__orderings) + + def __filter_disjunction(self, operations, values): + """Add a disjunction of several filters and several values to the query. + + This is implemented by duplicating queries and combining the + results later. + + Args: + operations: a string or list of strings. Each string contains a + property name and an operator to filter by. The operators + themselves must not require multiple queries to evaluate + (currently, this means that 'in' and '!=' are invalid). + + values: a value or list of filter values, normalized by + _normalize_query_parameter. + """ + if not isinstance(operations, (list, tuple)): + operations = [operations] + if not isinstance(values, (list, tuple)): + values = [values] + + new_query_sets = [] + for operation in operations: + if operation.lower().endswith('in') or operation.endswith('!='): + raise BadQueryError('Cannot use "in" or "!=" in a disjunction.') + for query_set in self.__query_sets: + for value in values: + new_query_set = copy.copy(query_set) + datastore._AddOrAppend(new_query_set, operation, value) + new_query_sets.append(new_query_set) + self.__query_sets = new_query_sets def filter(self, property_operator, value): """Add filter to query. @@ -1544,12 +1851,45 @@ class Query(_BaseQuery): Returns: Self to support method chaining. + + Raises: + PropertyError if invalid property is provided. """ - if isinstance(value, (list, tuple)): - raise BadValueError('Filtering on lists is not supported') + match = _FILTER_REGEX.match(property_operator) + prop = match.group(1) + if match.group(3) is not None: + operator = match.group(3) + else: + operator = '==' + + if self._model_class is None: + if prop != datastore_types._KEY_SPECIAL_PROPERTY: + raise BadQueryError( + 'Only %s filters are allowed on kindless queries.' % + datastore_types._KEY_SPECIAL_PROPERTY) + elif prop in self._model_class._unindexed_properties: + raise PropertyError('Property \'%s\' is not indexed' % prop) + + if operator.lower() == 'in': + if self._keys_only: + raise BadQueryError('Keys only queries do not support IN filters.') + elif not isinstance(value, (list, tuple)): + raise BadValueError('Argument to the "in" operator must be a list') + values = [_normalize_query_parameter(v) for v in value] + self.__filter_disjunction(prop + ' =', values) + else: + if isinstance(value, (list, tuple)): + raise BadValueError('Filtering on lists is not supported') + if operator == '!=': + if self._keys_only: + raise BadQueryError('Keys only queries do not support != filters.') + self.__filter_disjunction([prop + ' <', prop + ' >'], + _normalize_query_parameter(value)) + else: + value = _normalize_query_parameter(value) + for query_set in self.__query_sets: + datastore._AddOrAppend(query_set, property_operator, value) - value = _normalize_query_parameter(value) - datastore._AddOrAppend(self.__query_set, property_operator, value) return self def order(self, property): @@ -1565,7 +1905,7 @@ class Query(_BaseQuery): Self to support method chaining. Raises: - PropertyError if invalid property name is provided. + PropertyError if invalid property is provided. """ if property.startswith('-'): property = property[1:] @@ -1573,10 +1913,20 @@ class Query(_BaseQuery): else: order = datastore.Query.ASCENDING - if not issubclass(self._model_class, Expando): - if (property not in self._model_class.properties() and - property not in datastore_types._SPECIAL_PROPERTIES): - raise PropertyError('Invalid property name \'%s\'' % property) + if self._model_class is None: + if (property != datastore_types._KEY_SPECIAL_PROPERTY or + order != datastore.Query.ASCENDING): + raise BadQueryError( + 'Only %s ascending orders are supported on kindless queries' % + datastore_types._KEY_SPECIAL_PROPERTY) + else: + if not issubclass(self._model_class, Expando): + if (property not in self._model_class.properties() and + property not in datastore_types._SPECIAL_PROPERTIES): + raise PropertyError('Invalid property name \'%s\'' % property) + + if property in self._model_class._unindexed_properties: + raise PropertyError('Property \'%s\' is not indexed' % property) self.__orderings.append((property, order)) return self @@ -1624,11 +1974,28 @@ class GqlQuery(_BaseQuery): query_string: Properly formatted GQL query string. *args: Positional arguments used to bind numeric references in the query. **kwds: Dictionary-based arguments for named references. + + Raises: + PropertyError if the query filters or sorts on a property that's not + indexed. """ from google.appengine.ext import gql app = kwds.pop('_app', None) + self._proto_query = gql.GQL(query_string, _app=app) - super(GqlQuery, self).__init__(class_for_kind(self._proto_query._entity)) + if self._proto_query._entity is not None: + model_class = class_for_kind(self._proto_query._entity) + else: + model_class = None + super(GqlQuery, self).__init__(model_class, + keys_only=self._proto_query._keys_only) + + if model_class is not None: + for property, unused in (self._proto_query.filters().keys() + + self._proto_query.orderings()): + if property in model_class._unindexed_properties: + raise PropertyError('Property \'%s\' is not indexed' % property) + self.bind(*args, **kwds) def bind(self, *args, **kwds): @@ -1655,39 +2022,56 @@ class GqlQuery(_BaseQuery): def run(self): """Override _BaseQuery.run() so the LIMIT clause is handled properly.""" query_run = self._proto_query.Run(*self._args, **self._kwds) - return _QueryIterator(self._model_class, iter(query_run)) + if self._keys_only: + return query_run + else: + return _QueryIterator(self._model_class, iter(query_run)) def _get_query(self): return self._proto_query.Bind(self._args, self._kwds) -class TextProperty(Property): - """A string that can be longer than 500 bytes. +class UnindexedProperty(Property): + """A property that isn't indexed by either built-in or composite indices. - This type should be used for large text values to make sure the datastore - has good performance for queries. + TextProperty and BlobProperty derive from this class. """ + def __init__(self, *args, **kwds): + """Construct property. See the Property class for details. + + Raises: + ConfigurationError if indexed=True. + """ + self._require_parameter(kwds, 'indexed', False) + kwds['indexed'] = True + super(UnindexedProperty, self).__init__(*args, **kwds) def validate(self, value): - """Validate text property. + """Validate property. Returns: A valid value. Raises: - BadValueError if property is not instance of 'Text'. + BadValueError if property is not an instance of data_type. """ - if value is not None and not isinstance(value, Text): + if value is not None and not isinstance(value, self.data_type): try: - value = Text(value) + value = self.data_type(value) except TypeError, err: raise BadValueError('Property %s must be convertible ' - 'to a Text instance (%s)' % (self.name, err)) - value = super(TextProperty, self).validate(value) - if value is not None and not isinstance(value, Text): - raise BadValueError('Property %s must be a Text instance' % self.name) + 'to a %s instance (%s)' % + (self.name, self.data_type.__name__, err)) + value = super(UnindexedProperty, self).validate(value) + if value is not None and not isinstance(value, self.data_type): + raise BadValueError('Property %s must be a %s instance' % + (self.name, self.data_type.__name__)) return value + +class TextProperty(UnindexedProperty): + """A string that can be longer than 500 bytes.""" + data_type = Text @@ -1801,32 +2185,8 @@ class PostalAddressProperty(_CoercingProperty): data_type = PostalAddress -class BlobProperty(Property): - """A string that can be longer than 500 bytes. - - This type should be used for large binary values to make sure the datastore - has good performance for queries. - """ - - def validate(self, value): - """Validate blob property. - - Returns: - A valid value. - - Raises: - BadValueError if property is not instance of 'Blob'. - """ - if value is not None and not isinstance(value, Blob): - try: - value = Blob(value) - except TypeError, err: - raise BadValueError('Property %s must be convertible ' - 'to a Blob instance (%s)' % (self.name, err)) - value = super(BlobProperty, self).validate(value) - if value is not None and not isinstance(value, Blob): - raise BadValueError('Property %s must be a Blob instance' % self.name) - return value +class BlobProperty(UnindexedProperty): + """A byte string that can be longer than 500 bytes.""" data_type = Blob @@ -2181,9 +2541,15 @@ class BooleanProperty(Property): class UserProperty(Property): """A user property.""" - def __init__(self, verbose_name=None, name=None, - required=False, validator=None, choices=None, - auto_current_user=False, auto_current_user_add=False): + def __init__(self, + verbose_name=None, + name=None, + required=False, + validator=None, + choices=None, + auto_current_user=False, + auto_current_user_add=False, + indexed=True): """Initializes this Property with the given options. Note: this does *not* support the 'default' keyword argument. @@ -2200,11 +2566,13 @@ class UserProperty(Property): each time the entity is written to the datastore. auto_current_user_add: If true, the value is set to the current user the first time the entity is written to the datastore. + indexed: Whether property is indexed. """ super(UserProperty, self).__init__(verbose_name, name, required=required, validator=validator, - choices=choices) + choices=choices, + indexed=indexed) self.auto_current_user = auto_current_user self.auto_current_user_add = auto_current_user_add @@ -2249,7 +2617,6 @@ class UserProperty(Property): data_type = users.User - class ListProperty(Property): """A property that stores a list of things. @@ -2275,13 +2642,14 @@ class ListProperty(Property): raise TypeError('Item type should be a type object') if item_type not in _ALLOWED_PROPERTY_TYPES: raise ValueError('Item type %s is not acceptable' % item_type.__name__) - if 'required' in kwds and kwds['required'] is not True: - raise ValueError('List values must be required') + if issubclass(item_type, (Blob, Text)): + self._require_parameter(kwds, 'indexed', False) + kwds['indexed'] = True + self._require_parameter(kwds, 'required', True) if default is None: default = [] self.item_type = item_type super(ListProperty, self).__init__(verbose_name, - required=True, default=default, **kwds) @@ -2359,8 +2727,11 @@ class ListProperty(Property): Returns: validated list appropriate to save in the datastore. """ - return self.validate_list_contents( + value = self.validate_list_contents( super(ListProperty, self).get_value_for_datastore(model_instance)) + if self.validator: + self.validator(value) + return value class StringListProperty(ListProperty): @@ -2452,8 +2823,13 @@ class ReferenceProperty(Property): if self.collection_name is None: self.collection_name = '%s_set' % (model_class.__name__.lower()) - if hasattr(self.reference_class, self.collection_name): - raise DuplicatePropertyError('Class %s already has property %s' + existing_prop = getattr(self.reference_class, self.collection_name, None) + if existing_prop is not None: + if not (isinstance(existing_prop, _ReverseReferenceProperty) and + existing_prop._prop_name == property_name and + existing_prop._model.__name__ == model_class.__name__ and + existing_prop._model.__module__ == model_class.__module__): + raise DuplicatePropertyError('Class %s already has property %s ' % (self.reference_class.__name__, self.collection_name)) setattr(self.reference_class, @@ -2601,13 +2977,23 @@ class _ReverseReferenceProperty(Property): Constructor does not take standard values of other property types. Args: - model: Model that this property is a collection of. - property: Foreign property on referred model that points back to this - properties entity. + model: Model class that this property is a collection of. + property: Name of foreign property on referred model that points back + to this properties entity. """ self.__model = model self.__property = prop + @property + def _model(self): + """Internal helper to access the model class, read-only.""" + return self.__model + + @property + def _prop_name(self): + """Internal helper to access the property name, read-only.""" + return self.__property + def __get__(self, model_instance, model_class): """Fetches collection of model instances of this collection property.""" if model_instance is not None: @@ -2622,5 +3008,7 @@ class _ReverseReferenceProperty(Property): run_in_transaction = datastore.RunInTransaction +run_in_transaction_custom_retries = datastore.RunInTransactionCustomRetries RunInTransaction = run_in_transaction +RunInTransactionCustomRetries = run_in_transaction_custom_retries google_appengine/google/appengine/ext/db/__init__.py @@ -105,8 +105,9 @@ from google.appengine.ext import db def monkey_patch(name, bases, namespace): """A 'metaclass' for adding new methods to an existing class. - In this version, existing methods can't be overridden; this is by - design, to avoid accidents. + This shouldn't be used to override existing methods. However, + because loading this module (like loading any module) should be + idempotent, we don't assert that. Usage example: @@ -131,7 +132,6 @@ def monkey_patch(name, bases, namespace): base = bases[0] for name, value in namespace.iteritems(): if name not in ('__metaclass__', '__module__'): - assert name not in base.__dict__, "Won't override attribute %r" % (name,) setattr(base, name, value) return base google_appengine/google/appengine/ext/db/djangoforms.py @@ -87,9 +87,12 @@ class PolymorphicClass(db.PropertiedClass): itself so that it subclasses can quickly know what the root of their hierarchy is and what kind they are stored in. __class_hierarchy__: List of classes describing the new model's place - in the class hierarchy. The first element is always the root - element while the last element is the new class itself. For example: + in the class hierarchy in reverse MRO order. The first element is + always the root class while the last element is always the new class. + MRO documentation: http://www.python.org/download/releases/2.3/mro/ + + For example: class Foo(PolymorphicClass): ... class Bar(Foo): ... @@ -107,30 +110,29 @@ class PolymorphicClass(db.PropertiedClass): discriminator (the 'class' property of the entity) when loading from the datastore. """ - if name == 'PolyModel' or PolyModel not in bases: - db._initialize_properties(cls, name, bases, dct) - super(db.PropertiedClass, cls).__init__(name, bases, dct) - else: + if name == 'PolyModel': + super(PolymorphicClass, cls).__init__(name, bases, dct, map_kind=False) + return + + elif PolyModel in bases: + if getattr(cls, '__class_hierarchy__', None): + raise db.ConfigurationError(('%s cannot derive from PolyModel as ' + '__class_hierarchy__ is already defined.') % cls.__name__) + cls.__class_hierarchy__ = [cls] cls.__root_class__ = cls super(PolymorphicClass, cls).__init__(name, bases, dct) + else: + super(PolymorphicClass, cls).__init__(name, bases, dct, map_kind=False) - if name == 'PolyModel': - return + cls.__class_hierarchy__ = [c for c in reversed(cls.mro()) + if issubclass(c, PolyModel) and c != PolyModel] - if cls is not cls.__root_class__: - poly_class = None - for base in cls.__bases__: - if issubclass(base, PolyModel): - poly_class = base - break - else: + if cls.__class_hierarchy__[0] != cls.__root_class__: raise db.ConfigurationError( - "Polymorphic class '%s' does not inherit from PolyModel." - % cls.__name__) - - cls.__class_hierarchy__ = poly_class.__class_hierarchy__ + [cls] - else: - cls.__class_hierarchy__ = [cls] + '%s cannot be derived from both root classes %s and %s' % + (cls.__name__, + cls.__class_hierarchy__[0].__name__, + cls.__root_class__.__name__)) _class_map[cls.class_key()] = cls @@ -310,13 +312,16 @@ class PolyModel(db.Model): return super(PolyModel, cls).from_entity(entity) @classmethod - def all(cls): + def all(cls, **kwds): """Get all instance of a class hierarchy. + Args: + kwds: Keyword parameters passed on to Model.all. + Returns: Query with filter set to match this class' discriminator. """ - query = super(PolyModel, cls).all() + query = super(PolyModel, cls).all(**kwds) if cls != cls.__root_class__: query.filter(_CLASS_KEY_PROPERTY + ' =', cls.class_name()) return query google_appengine/google/appengine/ext/db/polymodel.py @@ -28,7 +28,6 @@ data stored. import calendar import datetime -import heapq import logging import re import time @@ -38,6 +37,7 @@ from google.appengine.api import datastore_errors from google.appengine.api import datastore_types from google.appengine.api import users +MultiQuery = datastore.MultiQuery LOG_LEVEL = logging.DEBUG - 1 @@ -77,7 +77,7 @@ class GQL(object): The syntax for SELECT is fairly straightforward: - SELECT * FROM <entity> + SELECT [* | __key__ ] [FROM <entity>] [WHERE <condition> [AND <condition> ...]] [ORDER BY <property> [ASC | DESC] [, <property> [ASC | DESC] ...]] [LIMIT [<offset>,]<count>] @@ -104,7 +104,7 @@ class GQL(object): Execute('SELECT * FROM Story WHERE Author = :1 AND Date > :2') - Named parameters Execute('SELECT * FROM Story WHERE Author = :author AND Date > :date') - - Literals (numbers, and strings) + - Literals (numbers, strings, booleans, and NULL) Execute('SELECT * FROM Story WHERE Author = \'James\'') Users are also given the option of doing type conversions to other datastore @@ -144,9 +144,8 @@ class GQL(object): simple types (strings, integers, floats). - SELECT * will return an iterable set of entries, but other operations (schema - queries, updates, inserts or field selections) will return alternative - result types. + SELECT * will return an iterable set of entities; SELECT __key__ will return + an iterable set of Keys. """ TOKENIZE_REGEX = re.compile(r""" @@ -161,7 +160,7 @@ class GQL(object): \S+ """, re.VERBOSE | re.IGNORECASE) - MAX_ALLOWABLE_QUERIES = 30 + MAX_ALLOWABLE_QUERIES = datastore.MAX_ALLOWABLE_QUERIES __ANCESTOR = -1 @@ -229,7 +228,8 @@ class GQL(object): query_count = 1 for i in xrange(query_count): - queries.append(datastore.Query(self._entity, _app=self.__app)) + queries.append(datastore.Query(self._entity, _app=self.__app, + keys_only=self._keys_only)) logging.log(LOG_LEVEL, 'Binding with %i positional args %s and %i keywords %s' @@ -327,11 +327,11 @@ class GQL(object): """Cast input values to Key() class using encoded string or tuple list.""" if not len(values) % 2: return datastore_types.Key.from_path(_app=self.__app, *values) - elif len(values) == 1 and isinstance(values[0], str): + elif len(values) == 1 and isinstance(values[0], basestring): return datastore_types.Key(values[0]) else: self.__CastError('KEY', values, - 'requires an even number of operands' + 'requires an even number of operands ' 'or a single encoded string') def __CastGeoPt(self, values): @@ -343,41 +343,120 @@ class GQL(object): def __CastUser(self, values): """Cast to User() class using the email address in values[0].""" if len(values) != 1: - self.__CastError(values, 'user', 'requires one and only one value') + self.__CastError('user', values, 'requires one and only one value') + elif values[0] is None: + self.__CastError('user', values, 'must be non-null') else: return users.User(email=values[0], _auth_domain=self.__auth_domain) + def __EncodeIfNeeded(self, value): + """Simple helper function to create an str from possibly unicode strings. + Args: + value: input string (should pass as an instance of str or unicode). + """ + if isinstance(value, unicode): + return value.encode('utf8') + else: + return value + def __CastDate(self, values): - """Cast date values to DATETIME() class using ISO string or tuple inputs.""" - try: - if len(values) == 1 and isinstance(values[0], str): - time_tuple = time.strptime(values[0], '%Y-%m-%d') - return datetime.datetime(*time_tuple[0:6]) + """Cast DATE values (year/month/day) from input (to datetime.datetime). + + Casts DATE input values formulated as ISO string or time tuple inputs. + + Args: + values: either a single string with ISO time representation or 3 + integer valued date tuple (year, month, day). + + Returns: + datetime.datetime value parsed from the input values. + """ + + if len(values) == 1: + value = self.__EncodeIfNeeded(values[0]) + if isinstance(value, str): + try: + time_tuple = time.strptime(value, '%Y-%m-%d')[0:6] + except ValueError, err: + self.__CastError('DATE', values, err) else: - return datetime.datetime(values[0], values[1], values[2], 0, 0, 0) + self.__CastError('DATE', values, 'Single input value not a string') + elif len(values) == 3: + time_tuple = (values[0], values[1], values[2], 0, 0, 0) + else: + self.__CastError('DATE', values, + 'function takes 1 string or 3 integer values') + + try: + return datetime.datetime(*time_tuple) except ValueError, err: self.__CastError('DATE', values, err) def __CastTime(self, values): - """Cast time values to DATETIME() class using ISO string or tuple inputs.""" - try: - if len(values) == 1 and isinstance(values[0], str): - time_tuple = time.strptime(values[0], '%H:%M:%S') + """Cast TIME values (hour/min/sec) from input (to datetime.datetime). + + Casts TIME input values formulated as ISO string or time tuple inputs. + + Args: + values: either a single string with ISO time representation or 1-4 + integer valued time tuple (hour), (hour, minute), + (hour, minute, second), (hour, minute, second, microsec). + + Returns: + datetime.datetime value parsed from the input values. + """ + if len(values) == 1: + value = self.__EncodeIfNeeded(values[0]) + if isinstance(value, str): + try: + time_tuple = time.strptime(value, '%H:%M:%S') + except ValueError, err: + self.__CastError('TIME', values, err) time_tuple = (1970, 1, 1) + time_tuple[3:] - return datetime.datetime(*time_tuple[0:6]) + time_tuple = time_tuple[0:6] + elif isinstance(value, int): + time_tuple = (1970, 1, 1, value) else: - return datetime.datetime(1970, 1, 1, *values) + self.__CastError('TIME', values, + 'Single input value not a string or integer hour') + elif len(values) <= 4: + time_tuple = (1970, 1, 1) + tuple(values) + else: + self.__CastError('TIME', values, + 'function takes 1 to 4 integers or 1 string') + + try: + return datetime.datetime(*time_tuple) except ValueError, err: self.__CastError('TIME', values, err) def __CastDatetime(self, values): - """Cast values to DATETIME() class using ISO string or tuple inputs.""" - try: - if len(values) == 1 and isinstance(values[0], str): - time_tuple = time.strptime(values[0], '%Y-%m-%d %H:%M:%S') - return datetime.datetime(*time_tuple[0:6]) + """Cast DATETIME values (string or tuple) from input (to datetime.datetime). + + Casts DATETIME input values formulated as ISO string or datetime tuple + inputs. + + Args: + values: either a single string with ISO representation or 3-7 + integer valued time tuple (year, month, day, ...). + + Returns: + datetime.datetime value parsed from the input values. + """ + if len(values) == 1: + value = self.__EncodeIfNeeded(values[0]) + if isinstance(value, str): + try: + time_tuple = time.strptime(str(value), '%Y-%m-%d %H:%M:%S')[0:6] + except ValueError, err: + self.__CastError('DATETIME', values, err) else: - return datetime.datetime(*values) + self.__CastError('DATETIME', values, 'Single input value not a string') + else: + time_tuple = values + + try: + return datetime.datetime(*time_tuple) except ValueError, err: self.__CastError('DATETIME', values, err) @@ -454,7 +533,7 @@ class GQL(object): raise datastore_errors.BadArgumentError( 'Missing argument for bind, requires argument #%i, ' 'but only has %i args.' % (reference, num_args)) - elif isinstance(reference, str): + elif isinstance(reference, basestring): if reference in keyword_args: return keyword_args[reference] else: @@ -476,6 +555,9 @@ class GQL(object): Raises: BadArgumentError if the filter is invalid (namely non-list with IN) """ + if condition.lower() in ('!=', 'in') and self._keys_only: + raise datastore_errors.BadQueryError( + 'Keys only queries do not support IN or != filters.') def CloneQueries(queries, n): """Do a full copy of the queries and append to the end of the queries. @@ -597,8 +679,13 @@ class GQL(object): """Return the result ordering list.""" return self.__orderings + def is_keys_only(self): + """Returns True if this query returns Keys, False if it returns Entities.""" + return self._keys_only + __iter__ = Run + __result_type_regex = re.compile(r'(\*|__key__)') __quoted_string_regex = re.compile(r'((?:\'[^\'\n\r]*\')+)') __ordinal_regex = re.compile(r':(\d+)$') __named_regex = re.compile(r':(\w+)$') @@ -707,7 +794,8 @@ class GQL(object): True if parsing completed okay. """ self.__Expect('SELECT') - self.__Expect('*') + result_type = self.__AcceptRegex(self.__result_type_regex) + self._keys_only = (result_type == '__key__') return self.__From() def __From(self): @@ -720,14 +808,16 @@ class GQL(object): Returns: True if parsing completed okay. """ - self.__Expect('FROM') - entity = self.__AcceptRegex(self.__identifier_regex) - if entity: - self._entity = entity - return self.__Where() + if self.__Accept('FROM'): + kind = self.__AcceptRegex(self.__identifier_regex) + if kind: + self._entity = kind + else: + self.__Error('Identifier Expected') + return False else: - self.__Error('Identifier Expected') - return False + self._entity = None + return self.__Where() def __Where(self): """Consume the WHERE cluase. @@ -842,8 +932,8 @@ class GQL(object): filter_rule = (self.__ANCESTOR, 'is') assert condition.lower() == 'is' - if condition.lower() != 'in' and operator == 'list': - sef.__Error('Only IN can process a list of values') + if operator == 'list' and condition.lower() != 'in': + self.__Error('Only IN can process a list of values') self.__filters.setdefault(filter_rule, []).append((operator, parameters)) return True @@ -921,6 +1011,9 @@ class GQL(object): if literal is not None: return Literal(literal) + + if self.__Accept('NULL'): + return Literal(None) else: return None @@ -1062,220 +1155,3 @@ class Literal(object): def __repr__(self): return 'Literal(%s)' % repr(self.__value) - - -class MultiQuery(datastore.Query): - """Class representing a GQL query requiring multiple datastore queries. - - This class is actually a subclass of datastore.Query as it is intended to act - like a normal Query object (supporting the same interface). - """ - - def __init__(self, bound_queries, orderings): - self.__bound_queries = bound_queries - self.__orderings = orderings - - def __str__(self): - res = 'MultiQuery: ' - for query in self.__bound_queries: - res = '%s %s' % (res, str(query)) - return res - - def Get(self, limit, offset=0): - """Get results of the query with a limit on the number of results. - - Args: - limit: maximum number of values to return. - offset: offset requested -- if nonzero, this will override the offset in - the original query - - Returns: - A list of entities with at most "limit" entries (less if the query - completes before reading limit values). - """ - count = 1 - result = [] - - iterator = self.Run() - - try: - for i in xrange(offset): - val = iterator.next() - except StopIteration: - pass - - try: - while count <= limit: - val = iterator.next() - result.append(val) - count += 1 - except StopIteration: - pass - return result - - class SortOrderEntity(object): - def __init__(self, entity_iterator, orderings): - self.__entity_iterator = entity_iterator - self.__entity = None - self.__min_max_value_cache = {} - try: - self.__entity = entity_iterator.next() - except StopIteration: - pass - else: - self.__orderings = orderings - - def __str__(self): - return str(self.__entity) - - def GetEntity(self): - return self.__entity - - def GetNext(self): - return MultiQuery.SortOrderEntity(self.__entity_iterator, - self.__orderings) - - def CmpProperties(self, that): - """Compare two entities and return their relative order. - - Compares self to that based on the current sort orderings and the - key orders between them. Returns negative, 0, or positive depending on - whether self is less, equal to, or greater than that. This - comparison returns as if all values were to be placed in ascending order - (highest value last). Only uses the sort orderings to compare (ignores - keys). - - Args: - self: SortOrderEntity - that: SortOrderEntity - - Returns: - Negative if self < that - Zero if self == that - Positive if self > that - """ - if not self.__entity: - return cmp(self.__entity, that.__entity) - - for (identifier, order) in self.__orderings: - value1 = self.__GetValueForId(self, identifier, order) - value2 = self.__GetValueForId(that, identifier, order) - - result = cmp(value1, value2) - if order == datastore.Query.DESCENDING: - result = -result - if result: - return result - return 0 - - def __GetValueForId(self, sort_order_entity, identifier, sort_order): - value = sort_order_entity.__entity[identifier] - entity_key = sort_order_entity.__entity.key() - if self.__min_max_value_cache.has_key((entity_key, identifier)): - value = self.__min_max_value_cache[(entity_key, identifier)] - elif isinstance(value, list): - if sort_order == datastore.Query.DESCENDING: - value = min(value) - else: - value = max(value) - self.__min_max_value_cache[(entity_key, identifier)] = value - - return value - - def __cmp__(self, that): - """Compare self to that w.r.t. values defined in the sort order. - - Compare an entity with another, using sort-order first, then the key - order to break ties. This can be used in a heap to have faster min-value - lookup. - - Args: - that: other entity to compare to - Returns: - negative: if self is less than that in sort order - zero: if self is equal to that in sort order - positive: if self is greater than that in sort order - """ - property_compare = self.CmpProperties(that) - if property_compare: - return property_compare - else: - return cmp(self.__entity.key(), that.__entity.key()) - - def Run(self): - """Return an iterable output with all results in order.""" - results = [] - count = 1 - for bound_query in self.__bound_queries: - logging.log(LOG_LEVEL, 'Running query #%i' % count) - results.append(bound_query.Run()) - count += 1 - - def IterateResults(results): - """Iterator function to return all results in sorted order. - - Iterate over the array of results, yielding the next element, in - sorted order. This function is destructive (results will be empty - when the operation is complete). - - Args: - results: list of result iterators to merge and iterate through - - Yields: - The next result in sorted order. - """ - result_heap = [] - for result in results: - heap_value = MultiQuery.SortOrderEntity(result, self.__orderings) - if heap_value.GetEntity(): - heapq.heappush(result_heap, heap_value) - - used_keys = set() - - while result_heap: - top_result = heapq.heappop(result_heap) - - results_to_push = [] - if top_result.GetEntity().key() not in used_keys: - yield top_result.GetEntity() - else: - pass - - used_keys.add(top_result.GetEntity().key()) - - results_to_push = [] - while result_heap: - next = heapq.heappop(result_heap) - if cmp(top_result, next): - results_to_push.append(next) - break - else: - results_to_push.append(next.GetNext()) - results_to_push.append(top_result.GetNext()) - - for popped_result in results_to_push: - if popped_result.GetEntity(): - heapq.heappush(result_heap, popped_result) - - return IterateResults(results) - - def Count(self, limit=None): - """Return the number of matched entities for this query. - - Will return the de-duplicated count of results. Will call the more - efficient Get() function if a limit is given. - - Args: - limit: maximum number of entries to count (for any result > limit, return - limit). - Returns: - count of the number of entries returned. - """ - if limit is None: - count = 0 - for value in self.Run(): - count += 1 - return count - else: - return len(self.Get(limit)) - google_appengine/google/appengine/ext/gql/__init__.py @@ -189,7 +189,6 @@ import xmlrpclib import zipfile import zlib -import django import neo_cs import neo_util import webob @@ -202,13 +201,11 @@ from google.appengine.api import memcache from google.appengine.api import urlfetch from google.appengine.api import users -from google.appengine.ext import admin from google.appengine.ext import bulkload from google.appengine.ext import db from google.appengine.ext import gql from google.appengine.ext import search from google.appengine.ext import webapp -from google.appengine.ext.webapp import template from google.appengine.runtime import apiproxy google_appengine/google/appengine/ext/preload/__init__.py @@ -14,385 +14,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -"""An apiproxy stub that calls a remote handler via HTTP. - -This allows easy remote access to the App Engine datastore, and potentially any -of the other App Engine APIs, using the same interface you use when accessing -the service locally. - -An example Python script: ---- -from google.appengine.ext import db -from google.appengine.ext import remote_api -from myapp import models -import getpass - -def auth_func(): - return (raw_input('Username:'), getpass.getpass('Password:')) - -remote_api.ConfigureRemoteDatastore('my-app', '/remote_api', auth_func) - -# Now you can access the remote datastore just as if your code was running on -# App Engine! - -houses = models.House.all().fetch(100) -for a_house in q: - a_house.doors += 1 -db.put(houses) ---- - -A few caveats: -- Where possible, avoid iterating over queries directly. Fetching as many - results as you will need is faster and more efficient. -- If you need to iterate, consider instead fetching items in batches with a sort - order and constructing a new query starting from where the previous one left - off. The __key__ pseudo-property can be used as a sort key for this purpose, - and does not even require a custom index if you are iterating over all - entities of a given type. -- Likewise, it's a good idea to put entities in batches. Instead of calling put - for each individual entity, accumulate them and put them in batches using - db.put(), if you can. -- Requests and responses are still limited to 1MB each, so if you have large - entities or try and fetch or put many of them at once, your requests may fail. -""" - - - - - -import os -import pickle -import sha -import sys -import thread -import threading -from google.appengine.api import apiproxy_stub_map -from google.appengine.datastore import datastore_pb -from google.appengine.ext.remote_api import remote_api_pb -from google.appengine.runtime import apiproxy_errors -from google.appengine.tools import appengine_rpc - - -def GetUserAgent(): - """Determines the value of the 'User-agent' header to use for HTTP requests. - - Returns: - String containing the 'user-agent' header value, which includes the SDK - version, the platform information, and the version of Python; - e.g., "remote_api/1.0.1 Darwin/9.2.0 Python/2.5.2". - """ - product_tokens = [] - - product_tokens.append("Google-remote_api/1.0") - - product_tokens.append(appengine_rpc.GetPlatformToken()) - - python_version = ".".join(str(i) for i in sys.version_info) - product_tokens.append("Python/%s" % python_version) - - return " ".join(product_tokens) - - -def GetSourceName(): - return "Google-remote_api-1.0" - - -class TransactionData(object): - """Encapsulates data about an individual transaction.""" - - def __init__(self, thread_id): - self.thread_id = thread_id - self.preconditions = {} - self.entities = {} - - -class RemoteStub(object): - """A stub for calling services on a remote server over HTTP. - - You can use this to stub out any service that the remote server supports. - """ - - def __init__(self, server, path): - """Constructs a new RemoteStub that communicates with the specified server. - - Args: - server: An instance of a subclass of - google.appengine.tools.appengine_rpc.AbstractRpcServer. - path: The path to the handler this stub should send requests to. - """ - self._server = server - self._path = path - - def MakeSyncCall(self, service, call, request, response): - request_pb = remote_api_pb.Request() - request_pb.set_service_name(service) - request_pb.set_method(call) - request_pb.mutable_request().set_contents(request.Encode()) - - response_pb = remote_api_pb.Response() - response_pb.ParseFromString(self._server.Send(self._path, - request_pb.Encode())) - - if response_pb.has_exception(): - raise pickle.loads(response_pb.exception().contents()) - else: - response.ParseFromString(response_pb.response().contents()) - - -class RemoteDatastoreStub(RemoteStub): - """A specialised stub for accessing the App Engine datastore remotely. - - A specialised stub is required because there are some datastore operations - that preserve state between calls. This stub makes queries possible. - Transactions on the remote datastore are unfortunately still impossible. - """ - - def __init__(self, server, path): - super(RemoteDatastoreStub, self).__init__(server, path) - self.__queries = {} - self.__transactions = {} - - self.__next_local_cursor = 1 - self.__local_cursor_lock = threading.Lock() - self.__next_local_tx = 1 - self.__local_tx_lock = threading.Lock() - - def MakeSyncCall(self, service, call, request, response): - assert service == 'datastore_v3' - - explanation = [] - assert request.IsInitialized(explanation), explanation - - handler = getattr(self, '_Dynamic_' + call, None) - if handler: - handler(request, response) - else: - super(RemoteDatastoreStub, self).MakeSyncCall(service, call, request, - response) - - assert response.IsInitialized(explanation), explanation - - def _Dynamic_RunQuery(self, query, query_result): - self.__local_cursor_lock.acquire() - try: - cursor_id = self.__next_local_cursor - self.__next_local_cursor += 1 - finally: - self.__local_cursor_lock.release() - self.__queries[cursor_id] = query - - query_result.mutable_cursor().set_cursor(cursor_id) - query_result.set_more_results(True) - - def _Dynamic_Next(self, next_request, query_result): - cursor = next_request.cursor().cursor() - if cursor not in self.__queries: - raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, - 'Cursor %d not found' % cursor) - query = self.__queries[cursor] - - if query is None: - query_result.set_more_results(False) - return - - request = datastore_pb.Query() - request.CopyFrom(query) - if request.has_limit(): - request.set_limit(min(request.limit(), next_request.count())) - else: - request.set_limit(next_request.count()) - - super(RemoteDatastoreStub, self).MakeSyncCall( - 'remote_datastore', 'RunQuery', request, query_result) - - query.set_offset(query.offset() + query_result.result_size()) - if query.has_limit(): - query.set_limit(query.limit() - query_result.result_size()) - if not query_result.more_results(): - self.__queries[cursor] = None - - def _Dynamic_Get(self, get_request, get_response): - txid = None - if get_request.has_transaction(): - txid = get_request.transaction().handle() - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - - keys = [(k, k.Encode()) for k in get_request.key_list()] - - new_request = datastore_pb.GetRequest() - for key, enckey in keys: - if enckey not in txdata.entities: - new_request.add_key().CopyFrom(key) - else: - new_request = get_request - - if new_request.key_size() > 0: - super(RemoteDatastoreStub, self).MakeSyncCall( - 'datastore_v3', 'Get', new_request, get_response) - - if txid is not None: - newkeys = new_request.key_list() - entities = get_response.entity_list() - for key, entity in zip(newkeys, entities): - entity_hash = None - if entity.has_entity(): - entity_hash = sha.new(entity.entity().Encode()).digest() - txdata.preconditions[key.Encode()] = (key, entity_hash) - - new_response = datastore_pb.GetResponse() - it = iter(get_response.entity_list()) - for key, enckey in keys: - if enckey in txdata.entities: - cached_entity = txdata.entities[enckey][1] - if cached_entity: - new_response.add_entity().mutable_entity().CopyFrom(cached_entity) - else: - new_response.add_entity() - else: - new_entity = it.next() - if new_entity.has_entity(): - assert new_entity.entity().key() == key - new_response.add_entity().CopyFrom(new_entity) - else: - new_response.add_entity() - get_response.CopyFrom(new_response) - - def _Dynamic_Put(self, put_request, put_response): - if put_request.has_transaction(): - entities = put_request.entity_list() - - requires_id = lambda x: x.id() == 0 and not x.has_name() - new_ents = [e for e in entities - if requires_id(e.key().path().element_list()[-1])] - id_request = remote_api_pb.PutRequest() - if new_ents: - for ent in new_ents: - e = id_request.add_entity() - e.mutable_key().CopyFrom(ent.key()) - e.mutable_entity_group() - id_response = datastore_pb.PutResponse() - super(RemoteDatastoreStub, self).MakeSyncCall( - 'remote_datastore', 'GetIDs', id_request, id_response) - assert id_request.entity_size() == id_response.key_size() - for key, ent in zip(id_response.key_list(), new_ents): - ent.mutable_key().CopyFrom(key) - ent.mutable_entity_group().add_element().CopyFrom( - key.path().element(0)) - - txid = put_request.transaction().handle() - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - for entity in entities: - txdata.entities[entity.key().Encode()] = (entity.key(), entity) - put_response.add_key().CopyFrom(entity.key()) - else: - super(RemoteDatastoreStub, self).MakeSyncCall( - 'datastore_v3', 'Put', put_request, put_response) - - def _Dynamic_Delete(self, delete_request, response): - if delete_request.has_transaction(): - txid = delete_request.transaction().handle() - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - for key in delete_request.key_list(): - txdata.entities[key.Encode()] = (key, None) - else: - super(RemoteDatastoreStub, self).MakeSyncCall( - 'datastore_v3', 'Delete', delete_request, response) - - def _Dynamic_BeginTransaction(self, request, transaction): - self.__local_tx_lock.acquire() - try: - txid = self.__next_local_tx - self.__transactions[txid] = TransactionData(thread.get_ident()) - self.__next_local_tx += 1 - finally: - self.__local_tx_lock.release() - transaction.set_handle(txid) - - def _Dynamic_Commit(self, transaction, transaction_response): - txid = transaction.handle() - if txid not in self.__transactions: - raise apiproxy_errors.ApplicationError( - datastore_pb.Error.BAD_REQUEST, - 'Transaction %d not found.' % (txid,)) - - txdata = self.__transactions[txid] - assert (txdata.thread_id == thread.get_ident(), - "Transactions are single-threaded.") - del self.__transactions[txid] - - tx = remote_api_pb.TransactionRequest() - for key, hash in txdata.preconditions.values(): - precond = tx.add_precondition() - precond.mutable_key().CopyFrom(key) - if hash: - precond.set_hash(hash) - - puts = tx.mutable_puts() - deletes = tx.mutable_deletes() - for key, entity in txdata.entities.values(): - if entity: - puts.add_entity().CopyFrom(entity) - else: - deletes.add_key().CopyFrom(key) - - super(RemoteDatastoreStub, self).MakeSyncCall( - 'remote_datastore', 'Transaction', - tx, datastore_pb.PutResponse()) - - def _Dynamic_Rollback(self, transaction, transaction_response): - txid = transaction.handle() - self.__local_tx_lock.acquire() - try: - if txid not in self.__transactions: - raise apiproxy_errors.ApplicationError( - datastore_pb.Error.BAD_REQUEST, - 'Transaction %d not found.' % (txid,)) - - assert (txdata[txid].thread_id == thread.get_ident(), - "Transactions are single-threaded.") - del self.__transactions[txid] - finally: - self.__local_tx_lock.release() - - def _Dynamic_CreateIndex(self, index, id_response): - raise apiproxy_errors.CapabilityDisabledError( - 'The remote datastore does not support index manipulation.') - - def _Dynamic_UpdateIndex(self, index, void): - raise apiproxy_errors.CapabilityDisabledError( - 'The remote datastore does not support index manipulation.') - - def _Dynamic_DeleteIndex(self, index, void): - raise apiproxy_errors.CapabilityDisabledError( - 'The remote datastore does not support index manipulation.') - - -def ConfigureRemoteDatastore(app_id, path, auth_func, servername=None): - """Does necessary setup to allow easy remote access to an AppEngine datastore. - - Args: - app_id: The app_id of your app, as declared in app.yaml. - path: The path to the remote_api handler for your app - (for example, '/remote_api'). - auth_func: A function that takes no arguments and returns a - (username, password) tuple. This will be called if your application - requires authentication to access the remote_api handler (it should!) - and you do not already have a valid auth cookie. - servername: The hostname your app is deployed on. Defaults to - <app_id>.appspot.com. - """ - if not servername: - servername = '%s.appspot.com' % (app_id,) - os.environ['APPLICATION_ID'] = app_id - apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() - server = appengine_rpc.HttpRpcServer(servername, auth_func, GetUserAgent(), - GetSourceName()) - stub = RemoteDatastoreStub(server, path) - apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub) google_appengine/google/appengine/ext/remote_api/__init__.py @@ -38,13 +38,24 @@ So unwise that the default handler insists on checking for itself. import google +import logging +import os import pickle import sha import wsgiref.handlers +import yaml + from google.appengine.api import api_base_pb from google.appengine.api import apiproxy_stub from google.appengine.api import apiproxy_stub_map +from google.appengine.api import mail_service_pb +from google.appengine.api import urlfetch_service_pb from google.appengine.api import users +from google.appengine.api.capabilities import capability_service_pb +from google.appengine.api.images import images_service_pb +from google.appengine.api.memcache import memcache_service_pb +from google.appengine.api.taskqueue import taskqueue_service_pb +from google.appengine.api.xmpp import xmpp_service_pb from google.appengine.datastore import datastore_pb from google.appengine.ext import webapp from google.appengine.ext.remote_api import remote_api_pb @@ -63,6 +74,19 @@ class RemoteDatastoreStub(apiproxy_stub.APIProxyStub): RunQuery that immediately returns the query results. """ + def __init__(self, service='datastore_v3', _test_stub_map=None): + """Constructor. + + Args: + service: The name of the service + _test_stub_map: An APIProxyStubMap to use for testing purposes. + """ + super(RemoteDatastoreStub, self).__init__(service) + if _test_stub_map: + self.__call = _test_stub_map.MakeSyncCall + else: + self.__call = apiproxy_stub_map.MakeSyncCall + def _Dynamic_RunQuery(self, request, response): """Handle a RunQuery request. @@ -70,13 +94,15 @@ class RemoteDatastoreStub(apiproxy_stub.APIProxyStub): of the Next request. """ runquery_response = datastore_pb.QueryResult() - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'RunQuery', - request, runquery_response) + self.__call('datastore_v3', 'RunQuery', request, runquery_response) + if runquery_response.result_size() > 0: + response.CopyFrom(runquery_response) + return + next_request = datastore_pb.NextRequest() next_request.mutable_cursor().CopyFrom(runquery_response.cursor()) next_request.set_count(request.limit()) - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Next', - next_request, response) + self.__call('datastore_v3', 'Next', next_request, response) def _Dynamic_Transaction(self, request, response): """Handle a Transaction request. @@ -88,8 +114,7 @@ class RemoteDatastoreStub(apiproxy_stub.APIProxyStub): transaction of its own to make the updates. """ tx = datastore_pb.Transaction() - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction', - api_base_pb.VoidProto(), tx) + self.__call('datastore_v3', 'BeginTransaction', api_base_pb.VoidProto(), tx) preconditions = request.precondition_list() if preconditions: @@ -99,8 +124,7 @@ class RemoteDatastoreStub(apiproxy_stub.APIProxyStub): key = get_request.add_key() key.CopyFrom(precondition.key()) get_response = datastore_pb.GetResponse() - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Get', get_request, - get_response) + self.__call('datastore_v3', 'Get', get_request, get_response) entities = get_response.entity_list() assert len(entities) == request.precondition_size() for precondition, entity in zip(preconditions, entities): @@ -118,17 +142,15 @@ class RemoteDatastoreStub(apiproxy_stub.APIProxyStub): if request.has_puts(): put_request = request.puts() put_request.mutable_transaction().CopyFrom(tx) - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Put', - put_request, response) + self.__call('datastore_v3', 'Put', put_request, response) if request.has_deletes(): delete_request = request.deletes() delete_request.mutable_transaction().CopyFrom(tx) - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Delete', - delete_request, api_base_pb.VoidProto()) + self.__call('datastore_v3', 'Delete', delete_request, + api_base_pb.VoidProto()) - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit', tx, - api_base_pb.VoidProto()) + self.__call('datastore_v3', 'Commit', tx, api_base_pb.VoidProto()) def _Dynamic_GetIDs(self, request, response): """Fetch unique IDs for a set of paths.""" @@ -140,28 +162,81 @@ class RemoteDatastoreStub(apiproxy_stub.APIProxyStub): assert lastpart.id() == 0 and not lastpart.has_name() tx = datastore_pb.Transaction() - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction', - api_base_pb.VoidProto(), tx) + self.__call('datastore_v3', 'BeginTransaction', api_base_pb.VoidProto(), tx) - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Put', request, response) + self.__call('datastore_v3', 'Put', request, response) - apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Rollback', tx, - api_base_pb.VoidProto()) + self.__call('datastore_v3', 'Rollback', tx, api_base_pb.VoidProto()) SERVICE_PB_MAP = { + 'capability_service': { + 'IsEnabled': (capability_service_pb.IsEnabledRequest, + capability_service_pb.IsEnabledResponse), + }, 'datastore_v3': { - 'Get': (datastore_pb.GetRequest, datastore_pb.GetResponse), - 'Put': (datastore_pb.PutRequest, datastore_pb.PutResponse), - 'Delete': (datastore_pb.DeleteRequest, datastore_pb.DeleteResponse), - 'Count': (datastore_pb.Query, api_base_pb.Integer64Proto), + 'Get': (datastore_pb.GetRequest, datastore_pb.GetResponse), + 'Put': (datastore_pb.PutRequest, datastore_pb.PutResponse), + 'Delete': (datastore_pb.DeleteRequest, datastore_pb.DeleteResponse), + 'Count': (datastore_pb.Query, api_base_pb.Integer64Proto), 'GetIndices': (api_base_pb.StringProto, datastore_pb.CompositeIndices), + 'AllocateIds':(datastore_pb.AllocateIdsRequest, + datastore_pb.AllocateIdsResponse), + }, + 'images': { + 'Transform': (images_service_pb.ImagesTransformRequest, + images_service_pb.ImagesTransformResponse), + 'Composite': (images_service_pb.ImagesCompositeRequest, + images_service_pb.ImagesCompositeResponse), + 'Histogram': (images_service_pb.ImagesHistogramRequest, + images_service_pb.ImagesHistogramResponse), + }, + 'mail': { + 'Send': (mail_service_pb.MailMessage, api_base_pb.VoidProto), + 'SendToAdmins': (mail_service_pb.MailMessage, api_base_pb.VoidProto), + }, + 'memcache': { + 'Get': (memcache_service_pb.MemcacheGetRequest, + memcache_service_pb.MemcacheGetResponse), + 'Set': (memcache_service_pb.MemcacheSetRequest, + memcache_service_pb.MemcacheSetResponse), + 'Delete': (memcache_service_pb.MemcacheDeleteRequest, + memcache_service_pb.MemcacheDeleteResponse), + 'Increment': (memcache_service_pb.MemcacheIncrementRequest, + memcache_service_pb.MemcacheIncrementResponse), + 'FlushAll': (memcache_service_pb.MemcacheFlushRequest, + memcache_service_pb.MemcacheFlushResponse), + 'Stats': (memcache_service_pb.MemcacheStatsRequest, + memcache_service_pb.MemcacheStatsResponse), + }, + 'taskqueue': { + 'Add': (taskqueue_service_pb.TaskQueueAddRequest, + taskqueue_service_pb.TaskQueueAddResponse), + 'UpdateQueue':(taskqueue_service_pb.TaskQueueUpdateQueueRequest, + taskqueue_service_pb.TaskQueueUpdateQueueResponse), + 'FetchQueues':(taskqueue_service_pb.TaskQueueFetchQueuesRequest, + taskqueue_service_pb.TaskQueueFetchQueuesResponse), + 'FetchQueueStats':( + taskqueue_service_pb.TaskQueueFetchQueueStatsRequest, + taskqueue_service_pb.TaskQueueFetchQueueStatsResponse), }, 'remote_datastore': { - 'RunQuery': (datastore_pb.Query, datastore_pb.QueryResult), + 'RunQuery': (datastore_pb.Query, datastore_pb.QueryResult), 'Transaction': (remote_api_pb.TransactionRequest, - datastore_pb.PutResponse), - 'GetIDs': (remote_api_pb.PutRequest, datastore_pb.PutResponse), + datastore_pb.PutResponse), + 'GetIDs': (remote_api_pb.PutRequest, datastore_pb.PutResponse), + }, + 'urlfetch': { + 'Fetch': (urlfetch_service_pb.URLFetchRequest, + urlfetch_service_pb.URLFetchResponse), + }, + 'xmpp': { + 'GetPresence': (xmpp_service_pb.PresenceRequest, + xmpp_service_pb.PresenceResponse), + 'SendMessage': (xmpp_service_pb.XmppMessageRequest, + xmpp_service_pb.XmppMessageResponse), + 'SendInvite': (xmpp_service_pb.XmppInviteRequest, + xmpp_service_pb.XmppInviteResponse), }, } @@ -183,6 +258,7 @@ class ApiCallHandler(webapp.RequestHandler): elif 'X-appcfg-api-version' not in self.request.headers: self.response.set_status(403) self.response.out.write("This request did not contain a necessary header") + self.response.headers['Content-Type'] = 'text/plain' return False return True @@ -192,8 +268,14 @@ class ApiCallHandler(webapp.RequestHandler): if not self.CheckIsAdmin(): return - page = self.InfoPage() - self.response.out.write(page) + rtok = self.request.get('rtok', '0') + app_info = { + 'app_id': os.environ['APPLICATION_ID'], + 'rtok': rtok + } + + self.response.headers['Content-Type'] = 'text/plain' + self.response.out.write(yaml.dump(app_info)) def post(self): """Handle POST requests by executing the API call.""" @@ -209,8 +291,13 @@ class ApiCallHandler(webapp.RequestHandler): response.mutable_response().set_contents(response_data.Encode()) self.response.set_status(200) except Exception, e: - self.response.set_status(500) + logging.exception('Exception while handling %s', request) + self.response.set_status(200) response.mutable_exception().set_contents(pickle.dumps(e)) + if isinstance(e, apiproxy_errors.ApplicationError): + application_error = response.mutable_application_error() + application_error.set_code(e.application_error) + application_error.set_detail(e.error_detail) self.response.out.write(response.Encode()) def ExecuteRequest(self, request): google_appengine/google/appengine/ext/remote_api/handler.py @@ -44,8 +44,9 @@ class Request(ProtocolBuffer.ProtocolMessage): self.service_name_ = x def clear_service_name(self): - self.has_service_name_ = 0 - self.service_name_ = "" + if self.has_service_name_: + self.has_service_name_ = 0 + self.service_name_ = "" def has_service_name(self): return self.has_service_name_ @@ -56,8 +57,9 @@ class Request(ProtocolBuffer.ProtocolMessage): self.method_ = x def clear_method(self): - self.has_method_ = 0 - self.method_ = "" + if self.has_method_: + self.has_method_ = 0 + self.method_ = "" def has_method(self): return self.has_method_ @@ -153,29 +155,144 @@ class Request(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kservice_name = 2 kmethod = 3 krequest = 4 - _TEXT = ( - "ErrorCode", - None, - "service_name", - "method", - "request", - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 2: "service_name", + 3: "method", + 4: "request", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.STRING, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) + + _STYLE = """""" + _STYLE_CONTENT_TYPE = """""" +class ApplicationError(ProtocolBuffer.ProtocolMessage): + has_code_ = 0 + code_ = 0 + has_detail_ = 0 + detail_ = "" + + def __init__(self, contents=None): + if contents is not None: self.MergeFromString(contents) + + def code(self): return self.code_ + + def set_code(self, x): + self.has_code_ = 1 + self.code_ = x + + def clear_code(self): + if self.has_code_: + self.has_code_ = 0 + self.code_ = 0 + + def has_code(self): return self.has_code_ + + def detail(self): return self.detail_ + + def set_detail(self, x): + self.has_detail_ = 1 + self.detail_ = x + + def clear_detail(self): + if self.has_detail_: + self.has_detail_ = 0 + self.detail_ = "" + + def has_detail(self): return self.has_detail_ + + + def MergeFrom(self, x): + assert x is not self + if (x.has_code()): self.set_code(x.code()) + if (x.has_detail()): self.set_detail(x.detail()) + + def Equals(self, x): + if x is self: return 1 + if self.has_code_ != x.has_code_: return 0 + if self.has_code_ and self.code_ != x.code_: return 0 + if self.has_detail_ != x.has_detail_: return 0 + if self.has_detail_ and self.detail_ != x.detail_: return 0 + return 1 + + def IsInitialized(self, debug_strs=None): + initialized = 1 + if (not self.has_code_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: code not set.') + if (not self.has_detail_): + initialized = 0 + if debug_strs is not None: + debug_strs.append('Required field: detail not set.') + return initialized + + def ByteSize(self): + n = 0 + n += self.lengthVarInt64(self.code_) + n += self.lengthString(len(self.detail_)) + return n + 2 + + def Clear(self): + self.clear_code() + self.clear_detail() + + def OutputUnchecked(self, out): + out.putVarInt32(8) + out.putVarInt32(self.code_) + out.putVarInt32(18) + out.putPrefixedString(self.detail_) - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.MAX_TYPE, + def TryMerge(self, d): + while d.avail() > 0: + tt = d.getVarInt32() + if tt == 8: + self.set_code(d.getVarInt32()) + continue + if tt == 18: + self.set_detail(d.getPrefixedString()) + continue + if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError + d.skipData(tt) - ProtocolBuffer.Encoder.STRING, - ProtocolBuffer.Encoder.STRING, + def __str__(self, prefix="", printElemNumber=0): + res="" + if self.has_code_: res+=prefix+("code: %s\n" % self.DebugFormatInt32(self.code_)) + if self.has_detail_: res+=prefix+("detail: %s\n" % self.DebugFormatString(self.detail_)) + return res - ProtocolBuffer.Encoder.STRING, - ) + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + + kcode = 1 + kdetail = 2 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "code", + 2: "detail", + }, 2) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.NUMERIC, + 2: ProtocolBuffer.Encoder.STRING, + }, 2, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -184,6 +301,10 @@ class Response(ProtocolBuffer.ProtocolMessage): response_ = None has_exception_ = 0 exception_ = None + has_application_error_ = 0 + application_error_ = None + has_java_exception_ = 0 + java_exception_ = None def __init__(self, contents=None): self.lazy_init_lock_ = thread.allocate_lock() @@ -201,8 +322,9 @@ class Response(ProtocolBuffer.ProtocolMessage): def mutable_response(self): self.has_response_ = 1; return self.response() def clear_response(self): - self.has_response_ = 0; - if self.response_ is not None: self.response_.Clear() + if self.has_response_: + self.has_response_ = 0; + if self.response_ is not None: self.response_.Clear() def has_response(self): return self.has_response_ @@ -218,16 +340,55 @@ class Response(ProtocolBuffer.ProtocolMessage): def mutable_exception(self): self.has_exception_ = 1; return self.exception() def clear_exception(self): - self.has_exception_ = 0; - if self.exception_ is not None: self.exception_.Clear() + if self.has_exception_: + self.has_exception_ = 0; + if self.exception_ is not None: self.exception_.Clear() def has_exception(self): return self.has_exception_ + def application_error(self): + if self.application_error_ is None: + self.lazy_init_lock_.acquire() + try: + if self.application_error_ is None: self.application_error_ = ApplicationError() + finally: + self.lazy_init_lock_.release() + return self.application_error_ + + def mutable_application_error(self): self.has_application_error_ = 1; return self.application_error() + + def clear_application_error(self): + if self.has_application_error_: + self.has_application_error_ = 0; + if self.application_error_ is not None: self.application_error_.Clear() + + def has_application_error(self): return self.has_application_error_ + + def java_exception(self): + if self.java_exception_ is None: + self.lazy_init_lock_.acquire() + try: + if self.java_exception_ is None: self.java_exception_ = RawMessage() + finally: + self.lazy_init_lock_.release() + return self.java_exception_ + + def mutable_java_exception(self): self.has_java_exception_ = 1; return self.java_exception() + + def clear_java_exception(self): + if self.has_java_exception_: + self.has_java_exception_ = 0; + if self.java_exception_ is not None: self.java_exception_.Clear() + + def has_java_exception(self): return self.has_java_exception_ + def MergeFrom(self, x): assert x is not self if (x.has_response()): self.mutable_response().MergeFrom(x.response()) if (x.has_exception()): self.mutable_exception().MergeFrom(x.exception()) + if (x.has_application_error()): self.mutable_application_error().MergeFrom(x.application_error()) + if (x.has_java_exception()): self.mutable_java_exception().MergeFrom(x.java_exception()) def Equals(self, x): if x is self: return 1 @@ -235,23 +396,33 @@ class Response(ProtocolBuffer.ProtocolMessage): if self.has_response_ and self.response_ != x.response_: return 0 if self.has_exception_ != x.has_exception_: return 0 if self.has_exception_ and self.exception_ != x.exception_: return 0 + if self.has_application_error_ != x.has_application_error_: return 0 + if self.has_application_error_ and self.application_error_ != x.application_error_: return 0 + if self.has_java_exception_ != x.has_java_exception_: return 0 + if self.has_java_exception_ and self.java_exception_ != x.java_exception_: return 0 return 1 def IsInitialized(self, debug_strs=None): initialized = 1 if (self.has_response_ and not self.response_.IsInitialized(debug_strs)): initialized = 0 if (self.has_exception_ and not self.exception_.IsInitialized(debug_strs)): initialized = 0 + if (self.has_application_error_ and not self.application_error_.IsInitialized(debug_strs)): initialized = 0 + if (self.has_java_exception_ and not self.java_exception_.IsInitialized(debug_strs)): initialized = 0 return initialized def ByteSize(self): n = 0 if (self.has_response_): n += 1 + self.lengthString(self.response_.ByteSize()) if (self.has_exception_): n += 1 + self.lengthString(self.exception_.ByteSize()) + if (self.has_application_error_): n += 1 + self.lengthString(self.application_error_.ByteSize()) + if (self.has_java_exception_): n += 1 + self.lengthString(self.java_exception_.ByteSize()) return n + 0 def Clear(self): self.clear_response() self.clear_exception() + self.clear_application_error() + self.clear_java_exception() def OutputUnchecked(self, out): if (self.has_response_): @@ -262,6 +433,14 @@ class Response(ProtocolBuffer.ProtocolMessage): out.putVarInt32(18) out.putVarInt32(self.exception_.ByteSize()) self.exception_.OutputUnchecked(out) + if (self.has_application_error_): + out.putVarInt32(26) + out.putVarInt32(self.application_error_.ByteSize()) + self.application_error_.OutputUnchecked(out) + if (self.has_java_exception_): + out.putVarInt32(34) + out.putVarInt32(self.java_exception_.ByteSize()) + self.java_exception_.OutputUnchecked(out) def TryMerge(self, d): while d.avail() > 0: @@ -278,6 +457,18 @@ class Response(ProtocolBuffer.ProtocolMessage): d.skip(length) self.mutable_exception().TryMerge(tmp) continue + if tt == 26: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_application_error().TryMerge(tmp) + continue + if tt == 34: + length = d.getVarInt32() + tmp = ProtocolBuffer.Decoder(d.buffer(), d.pos(), d.pos() + length) + d.skip(length) + self.mutable_java_exception().TryMerge(tmp) + continue if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError d.skipData(tt) @@ -292,24 +483,40 @@ class Response(ProtocolBuffer.ProtocolMessage): res+=prefix+"exception <\n" res+=self.exception_.__str__(prefix + " ", printElemNumber) res+=prefix+">\n" + if self.has_application_error_: + res+=prefix+"application_error <\n" + res+=self.application_error_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" + if self.has_java_exception_: + res+=prefix+"java_exception <\n" + res+=self.java_exception_.__str__(prefix + " ", printElemNumber) + res+=prefix+">\n" return res - kresponse = 1 - kexception = 2 - - _TEXT = ( - "ErrorCode", - "response", - "exception", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STRING, - ProtocolBuffer.Encoder.STRING, + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) - ) + kresponse = 1 + kexception = 2 + kapplication_error = 3 + kjava_exception = 4 + + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "response", + 2: "exception", + 3: "application_error", + 4: "java_exception", + }, 4) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STRING, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.STRING, + }, 4, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" @@ -337,8 +544,9 @@ class TransactionRequest_Precondition(ProtocolBuffer.ProtocolMessage): self.hash_ = x def clear_hash(self): - self.has_hash_ = 0 - self.hash_ = "" + if self.has_hash_: + self.has_hash_ = 0 + self.hash_ = "" def has_hash(self): return self.has_hash_ @@ -448,8 +656,9 @@ class TransactionRequest(ProtocolBuffer.ProtocolMessage): def mutable_puts(self): self.has_puts_ = 1; return self.puts() def clear_puts(self): - self.has_puts_ = 0; - if self.puts_ is not None: self.puts_.Clear() + if self.has_puts_: + self.has_puts_ = 0; + if self.puts_ is not None: self.puts_.Clear() def has_puts(self): return self.has_puts_ @@ -465,8 +674,9 @@ class TransactionRequest(ProtocolBuffer.ProtocolMessage): def mutable_deletes(self): self.has_deletes_ = 1; return self.deletes() def clear_deletes(self): - self.has_deletes_ = 0; - if self.deletes_ is not None: self.deletes_.Clear() + if self.has_deletes_: + self.has_deletes_ = 0; + if self.deletes_ is not None: self.deletes_.Clear() def has_deletes(self): return self.has_deletes_ @@ -565,36 +775,35 @@ class TransactionRequest(ProtocolBuffer.ProtocolMessage): res+=prefix+">\n" return res + + def _BuildTagLookupTable(sparse, maxtag, default=None): + return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)]) + kPreconditionGroup = 1 kPreconditionkey = 2 kPreconditionhash = 3 kputs = 4 kdeletes = 5 - _TEXT = ( - "ErrorCode", - "Precondition", - "key", - "hash", - "puts", - "deletes", - ) - - _TYPES = ( - ProtocolBuffer.Encoder.NUMERIC, - ProtocolBuffer.Encoder.STARTGROUP, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ProtocolBuffer.Encoder.STRING, - - ) + _TEXT = _BuildTagLookupTable({ + 0: "ErrorCode", + 1: "Precondition", + 2: "key", + 3: "hash", + 4: "puts", + 5: "deletes", + }, 5) + + _TYPES = _BuildTagLookupTable({ + 0: ProtocolBuffer.Encoder.NUMERIC, + 1: ProtocolBuffer.Encoder.STARTGROUP, + 2: ProtocolBuffer.Encoder.STRING, + 3: ProtocolBuffer.Encoder.STRING, + 4: ProtocolBuffer.Encoder.STRING, + 5: ProtocolBuffer.Encoder.STRING, + }, 5, ProtocolBuffer.Encoder.MAX_TYPE) _STYLE = """""" _STYLE_CONTENT_TYPE = """""" -__all__ = ['Request','Response','TransactionRequest','TransactionRequest_Precondition'] +__all__ = ['Request','ApplicationError','Response','TransactionRequest','TransactionRequest_Precondition'] google_appengine/google/appengine/ext/remote_api/remote_api_pb.py @@ -46,6 +46,40 @@ orders, e.g.: The full text index is stored in a property named __searchable_text_index. +Specifying multiple indexes and properties to index +--------------------------------------------------- + +By default, one index is created with all string properties. You can define +multiple indexes and specify which properties should be indexed for each by +overriding SearchableProperties() method of model.SearchableModel, for example: + + class Article(search.SearchableModel): + @classmethod + def SearchableProperties(cls): + return [['book', 'author'], ['book']] + +In this example, two indexes will be maintained - one that includes 'book' and +'author' properties, and another one for 'book' property only. They will be +stored in properties named __searchable_text_index_book_author and +__searchable_text_index_book respectively. Note that the index that includes +all properties will not be created unless added explicitly like this: + + @classmethod + def SearchableProperties(cls): + return [['book', 'author'], ['book'], search.ALL_PROPERTIES] + +The default return value of SearchableProperties() is [search.ALL_PROPERTIES] +(one index, all properties). + +To search using a custom-defined index, pass its definition +in 'properties' parameter of 'search': + + Article.all().search('Lem', properties=['book', 'author']) + +Note that the order of properties in the list matters. + +Adding indexes to index.yaml +----------------------------- In general, if you just want to provide full text search, you *don't* need to add any extra indexes to your index.yaml. However, if you want to use search() @@ -60,6 +94,9 @@ example: direction: desc ... +Similarly, if you created a custom index (see above), use the name of the +property it's stored in, e.g. __searchable_text_index_book_author. + Note that using SearchableModel will noticeable increase the latency of save() operations, since it writes an index row for each indexable word. This also means that the latency of save() will increase roughly with the size of the @@ -79,6 +116,8 @@ from google.appengine.api import datastore_types from google.appengine.ext import db from google.appengine.datastore import datastore_pb +ALL_PROPERTIES = [] + class SearchableEntity(datastore.Entity): """A subclass of datastore.Entity that supports full text indexing. @@ -124,6 +163,8 @@ class SearchableEntity(datastore.Entity): _word_delimiter_regex = re.compile('[' + re.escape(string.punctuation) + ']') + _searchable_properties = [ALL_PROPERTIES] + def __init__(self, kind_or_entity, word_delimiter_regex=None, *args, **kwargs): """Constructor. May be called as a copy constructor. @@ -143,6 +184,10 @@ class SearchableEntity(datastore.Entity): self._word_delimiter_regex = word_delimiter_regex if isinstance(kind_or_entity, datastore.Entity): self._Entity__key = kind_or_entity._Entity__key + self._Entity__unindexed_properties = frozenset(kind_or_entity.unindexed_properties()) + if isinstance(kind_or_entity, SearchableEntity): + if getattr(kind_or_entity, '_searchable_properties', None) is not None: + self._searchable_properties = kind_or_entity._searchable_properties self.update(kind_or_entity) else: super(SearchableEntity, self).__init__(kind_or_entity, *args, **kwargs) @@ -153,22 +198,33 @@ class SearchableEntity(datastore.Entity): Returns: entity_pb.Entity """ - if SearchableEntity._FULL_TEXT_INDEX_PROPERTY in self: - del self[SearchableEntity._FULL_TEXT_INDEX_PROPERTY] - - index = set() - for (name, values) in self.items(): - if not isinstance(values, list): - values = [values] - if (isinstance(values[0], basestring) and - not isinstance(values[0], datastore_types.Blob)): - for value in values: - index.update(SearchableEntity._FullTextIndex( - value, self._word_delimiter_regex)) - - index_list = list(index) - if index_list: - self[SearchableEntity._FULL_TEXT_INDEX_PROPERTY] = index_list + for properties_to_index in self._searchable_properties: + index_property_name = SearchableEntity.IndexPropertyName(properties_to_index) + if index_property_name in self: + del self[index_property_name] + + + if not properties_to_index: + properties_to_index = self.keys() + + index = set() + for name in properties_to_index: + if not self.has_key(name): + continue + + values = self[name] + if not isinstance(values, list): + values = [values] + + if (isinstance(values[0], basestring) and + not isinstance(values[0], datastore_types.Blob)): + for value in values: + index.update(SearchableEntity._FullTextIndex( + value, self._word_delimiter_regex)) + + index_list = list(index) + if index_list: + self[index_property_name] = index_list return super(SearchableEntity, self)._ToPb() @@ -205,6 +261,16 @@ class SearchableEntity(datastore.Entity): return words + @classmethod + def IndexPropertyName(cls, properties): + """Given index definition, returns the name of the property to put it in.""" + name = SearchableEntity._FULL_TEXT_INDEX_PROPERTY + + if properties: + name += '_' + '_'.join(properties) + + return name + class SearchableQuery(datastore.Query): """A subclass of datastore.Query that supports full text search. @@ -213,7 +279,8 @@ class SearchableQuery(datastore.Query): SearchableEntity or SearchableModel classes. """ - def Search(self, search_query, word_delimiter_regex=None): + def Search(self, search_query, word_delimiter_regex=None, + properties=ALL_PROPERTIES): """Add a search query. This may be combined with filters. Note that keywords in the search query will be silently dropped if they @@ -229,28 +296,27 @@ class SearchableQuery(datastore.Query): datastore_types.ValidateString(search_query, 'search query') self._search_query = search_query self._word_delimiter_regex = word_delimiter_regex + self._properties = properties return self - def _ToPb(self, limit=None, offset=None): + def _ToPb(self, *args, **kwds): """Adds filters for the search query, then delegates to the superclass. - Raises BadFilterError if a filter on the index property already exists. - - Args: - # an upper bound on the number of results returned by the query. - limit: int - # number of results that match the query to skip. limit is applied - # after the offset is fulfilled. - offset: int + Mimics Query._ToPb()'s signature. Raises BadFilterError if a filter on the + index property already exists. Returns: datastore_pb.Query """ - if SearchableEntity._FULL_TEXT_INDEX_PROPERTY in self: + + properties = getattr(self, "_properties", ALL_PROPERTIES) + + index_property_name = SearchableEntity.IndexPropertyName(properties) + if index_property_name in self: raise datastore_errors.BadFilterError( - '%s is a reserved name.' % SearchableEntity._FULL_TEXT_INDEX_PROPERTY) + '%s is a reserved name.' % index_property_name) - pb = super(SearchableQuery, self)._ToPb(limit=limit, offset=offset) + pb = super(SearchableQuery, self)._ToPb(*args, **kwds) if hasattr(self, '_search_query'): keywords = SearchableEntity._FullTextIndex( @@ -259,24 +325,51 @@ class SearchableQuery(datastore.Query): filter = pb.add_filter() filter.set_op(datastore_pb.Query_Filter.EQUAL) prop = filter.add_property() - prop.set_name(SearchableEntity._FULL_TEXT_INDEX_PROPERTY) + prop.set_name(index_property_name) + prop.set_multiple(len(keywords) > 1) prop.mutable_value().set_stringvalue(unicode(keyword).encode('utf-8')) return pb +class SearchableMultiQuery(datastore.MultiQuery): + """A multiquery that supports Search() by searching subqueries.""" + + def Search(self, *args, **kwargs): + """Add a search query, by trying to add it to all subqueries. + + Args: + args: Passed to Search on each subquery. + kwargs: Passed to Search on each subquery. + + Returns: + self for consistency with SearchableQuery. + """ + for q in self: + q.Search(*args, **kwargs) + return self + + class SearchableModel(db.Model): """A subclass of db.Model that supports full text search and indexing. Automatically indexes all string-based properties. To search, use the all() method to get a SearchableModel.Query, then use its search() method. + + Override SearchableProperties() to define properties to index and/or multiple + indexes (see the file's comment). """ + @classmethod + def SearchableProperties(cls): + return [ALL_PROPERTIES] + class Query(db.Query): """A subclass of db.Query that supports full text search.""" _search_query = None + _properties = None - def search(self, search_query): + def search(self, search_query, properties=ALL_PROPERTIES): """Adds a full text search to this query. Args: @@ -286,20 +379,31 @@ class SearchableModel(db.Model): self """ self._search_query = search_query + self._properties = properties + + if self._properties not in getattr(self, '_searchable_properties', [ALL_PROPERTIES]): + raise datastore_errors.BadFilterError( + '%s does not have a corresponding index. Please add it to' + 'the SEARCHABLE_PROPERTIES list' % self._properties) + return self def _get_query(self): """Wraps db.Query._get_query() and injects SearchableQuery.""" - query = db.Query._get_query(self, _query_class=SearchableQuery) + query = db.Query._get_query(self, + _query_class=SearchableQuery, + _multi_query_class=SearchableMultiQuery) if self._search_query: - query.Search(self._search_query) + query.Search(self._search_query, properties=self._properties) return query def _populate_internal_entity(self): """Wraps db.Model._populate_internal_entity() and injects SearchableEntity.""" - return db.Model._populate_internal_entity(self, - _entity_class=SearchableEntity) + entity = db.Model._populate_internal_entity(self, + _entity_class=SearchableEntity) + entity._searchable_properties = self.SearchableProperties() + return entity @classmethod def from_entity(cls, entity): @@ -311,4 +415,6 @@ class SearchableModel(db.Model): @classmethod def all(cls): """Returns a SearchableModel.Query for this kind.""" - return SearchableModel.Query(cls) + query = SearchableModel.Query(cls) + query._searchable_properties = cls.SearchableProperties() + return query google_appengine/google/appengine/ext/search/__init__.py @@ -96,6 +96,9 @@ class Request(webob.Request): You can access parsed query and POST values with the get() method; do not parse the query string yourself. """ + + request_body_tempfile_limit = 0 + uri = property(lambda self: self.url) query = property(lambda self: self.query_string) @@ -244,6 +247,9 @@ class Response(object): except UnicodeError, e: logging.warning('Response written is not UTF-8: %s', e) + if (self.headers.get('Cache-Control') == 'no-cache' and + not self.headers.get('Expires')): + self.headers['Expires'] = 'Fri, 01 Jan 1990 00:00:00 GMT' self.headers['Content-Length'] = str(len(body)) write = start_response('%d %s' % self.__status, self.__wsgi_headers) write(body) @@ -460,6 +466,9 @@ class WSGIApplication(object): The URL mapping is first-match based on the list ordering. """ + REQUEST_CLASS = Request + RESPONSE_CLASS = Response + def __init__(self, url_mapping, debug=False): """Initializes this application with the given URL mapping. @@ -474,8 +483,8 @@ class WSGIApplication(object): def __call__(self, environ, start_response): """Called by WSGI when a request comes in.""" - request = Request(environ) - response = Response() + request = self.REQUEST_CLASS(environ) + response = self.RESPONSE_CLASS() WSGIApplication.active_instance = self google_appengine/google/appengine/ext/webapp/__init__.py @@ -21,13 +21,18 @@ -__all__ = ["login_required", "run_wsgi_app"] +__all__ = ['login_required', + 'run_wsgi_app', + 'add_wsgi_middleware', + 'run_bare_wsgi_app', + ] import os import sys import wsgiref.util from google.appengine.api import users +from google.appengine.api import lib_config from google.appengine.ext import webapp @@ -38,7 +43,7 @@ def login_required(handler_method): @login_required def get(self): - user = users.GetCurrentUser(self) + user = users.get_current_user(self) self.response.out.write('Hello, ' + user.nickname()) We will redirect to a login page if the user is not logged in. We always @@ -49,22 +54,56 @@ def login_required(handler_method): if self.request.method != 'GET': raise webapp.Error('The check_login decorator can only be used for GET ' 'requests') - user = users.GetCurrentUser() + user = users.get_current_user() if not user: - self.redirect(users.CreateLoginURL(self.request.uri)) + self.redirect(users.create_login_url(self.request.uri)) return else: handler_method(self, *args) return check_login +_config_handle = lib_config.register( + 'webapp', + {'add_wsgi_middleware': lambda app: app}) + + def run_wsgi_app(application): """Runs your WSGI-compliant application object in a CGI environment. Compared to wsgiref.handlers.CGIHandler().run(application), this function takes some shortcuts. Those are possible because the app server makes stronger promises than the CGI standard. + + Also, this function may wrap custom WSGI middleware around the + application. (You can use run_bare_wsgi_app() to run an application + without adding WSGI middleware, and add_wsgi_middleware() to wrap + the configured WSGI middleware around an application without running + it. This function is merely a convenient combination of the latter + two.) + + To configure custom WSGI middleware, define a function + webapp_add_wsgi_middleware(app) to your appengine_config.py file in + your application root directory: + + def webapp_add_wsgi_middleware(app): + app = MiddleWareClassOne(app) + app = MiddleWareClassTwo(app) + return app + + You must import the middleware classes elsewhere in the file. If + the function is not found, no WSGI middleware is added. """ + run_bare_wsgi_app(add_wsgi_middleware(application)) + + +def add_wsgi_middleware(application): + """Wrap WSGI middleware around a WSGI application object.""" + return _config_handle.add_wsgi_middleware(application) + + +def run_bare_wsgi_app(application): + """Like run_wsgi_app() but doesn't add WSGI middleware.""" env = dict(os.environ) env["wsgi.input"] = sys.stdin env["wsgi.errors"] = sys.stderr google_appengine/google/appengine/ext/webapp/util.py @@ -127,10 +127,11 @@ class RPC(apiproxy_rpc.RPC): _apphosting_runtime___python__apiproxy.MakeCall( self.package, self.call, e.buffer(), self.__result_dict, - self.__MakeCallDone, self) + self.__MakeCallDone, self, deadline=(self.deadline or -1)) def __MakeCallDone(self): self.__state = RPC.FINISHING + self.cpu_usage_mcycles = self.__result_dict['cpu_usage_mcycles'] if self.__result_dict['error'] == APPLICATION_ERROR: self.__exception = apiproxy_errors.ApplicationError( self.__result_dict['application_error'], google_appengine/google/appengine/runtime/apiproxy.py @@ -49,6 +49,7 @@ class ApplicationError(Error): def __init__(self, application_error, error_detail=''): self.application_error = application_error self.error_detail = error_detail + Error.__init__(self, application_error) def __str__(self): return 'ApplicationError: %d %s' % (self.application_error, google_appengine/google/appengine/runtime/apiproxy_errors.py @@ -30,18 +30,19 @@ files, and commit or rollback the transaction. import calendar -import cStringIO import datetime import getpass import logging import mimetypes import optparse import os +import random import re import sha import sys import tempfile import time +import urllib import urllib2 import google @@ -49,29 +50,40 @@ import yaml from google.appengine.cron import groctimespecification from google.appengine.api import appinfo from google.appengine.api import croninfo +from google.appengine.api import queueinfo from google.appengine.api import validation from google.appengine.api import yaml_errors from google.appengine.api import yaml_object from google.appengine.datastore import datastore_index from google.appengine.tools import appengine_rpc +from google.appengine.tools import bulkloader MAX_FILES_TO_CLONE = 100 -LIST_DELIMITER = "\n" -TUPLE_DELIMITER = "|" +LIST_DELIMITER = '\n' +TUPLE_DELIMITER = '|' -VERSION_FILE = "../VERSION" +VERSION_FILE = '../VERSION' UPDATE_CHECK_TIMEOUT = 3 -NAG_FILE = ".appcfg_nag" +NAG_FILE = '.appcfg_nag' MAX_LOG_LEVEL = 4 +MAX_BATCH_SIZE = 1000000 +MAX_BATCH_COUNT = 100 +MAX_BATCH_FILE_SIZE = 200000 +BATCH_OVERHEAD = 500 + verbosity = 1 -appinfo.AppInfoExternal.ATTRIBUTES[appinfo.RUNTIME] = "python" +appinfo.AppInfoExternal.ATTRIBUTES[appinfo.RUNTIME] = 'python' +_api_versions = os.environ.get('GOOGLE_TEST_API_VERSIONS', '1') +_options = validation.Options(*_api_versions.split(',')) +appinfo.AppInfoExternal.ATTRIBUTES[appinfo.API_VERSION] = _options +del _api_versions, _options def StatusUpdate(msg): @@ -102,9 +114,9 @@ def GetMimeTypeIfStaticFile(config, filename): """ for handler in config.handlers: handler_type = handler.GetHandlerType() - if handler_type in ("static_dir", "static_files"): - if handler_type == "static_dir": - regex = os.path.join(re.escape(handler.GetHandler()), ".*") + if handler_type in ('static_dir', 'static_files'): + if handler_type == 'static_dir': + regex = os.path.join(re.escape(handler.GetHandler()), '.*') else: regex = handler.upload if re.match(regex, filename): @@ -113,8 +125,8 @@ def GetMimeTypeIfStaticFile(config, filename): else: guess = mimetypes.guess_type(filename)[0] if guess is None: - default = "application/octet-stream" - print >>sys.stderr, ("Could not guess mimetype for %s. Using %s." + default = 'application/octet-stream' + print >>sys.stderr, ('Could not guess mimetype for %s. Using %s.' % (filename, default)) return default return guess @@ -149,8 +161,8 @@ class NagFile(validation.Validated): """ ATTRIBUTES = { - "timestamp": validation.TYPE_FLOAT, - "opt_in": validation.Optional(validation.TYPE_BOOL), + 'timestamp': validation.TYPE_FLOAT, + 'opt_in': validation.Optional(validation.TYPE_BOOL), } @staticmethod @@ -179,10 +191,10 @@ def GetVersionObject(isfile=os.path.isfile, open_fn=open): version_filename = os.path.join(os.path.dirname(google.__file__), VERSION_FILE) if not isfile(version_filename): - logging.error("Could not find version file at %s", version_filename) + logging.error('Could not find version file at %s', version_filename) return None - version_fh = open_fn(version_filename, "r") + version_fh = open_fn(version_filename, 'r') try: version = yaml.safe_load(version_fh) finally: @@ -191,6 +203,49 @@ def GetVersionObject(isfile=os.path.isfile, open_fn=open): return version +def RetryWithBackoff(initial_delay, backoff_factor, max_tries, callable_func): + """Calls a function multiple times, backing off more and more each time. + + Args: + initial_delay: Initial delay after first try, in seconds. + backoff_factor: Delay will be multiplied by this factor after each try. + max_tries: Maximum number of tries. + callable_func: The method to call, will pass no arguments. + + Returns: + True if the function succeded in one of its tries. + + Raises: + Whatever the function raises--an exception will immediately stop retries. + """ + delay = initial_delay + while not callable_func() and max_tries > 0: + StatusUpdate('Will check again in %s seconds.' % delay) + time.sleep(delay) + delay *= backoff_factor + max_tries -= 1 + return max_tries > 0 + + +def _VersionList(release): + """Parse a version string into a list of ints. + + Args: + release: The 'release' version, e.g. '1.2.4'. + (Due to YAML parsing this may also be an int or float.) + + Returns: + A list of ints corresponding to the parts of the version string + between periods. Example: + '1.2.4' -> [1, 2, 4] + '1.2.3.4' -> [1, 2, 3, 4] + + Raises: + ValueError if not all the parts are valid integers. + """ + return [int(part) for part in str(release).split('.')] + + class UpdateCheck(object): """Determines if the local SDK is the latest version. @@ -233,13 +288,13 @@ class UpdateCheck(object): @staticmethod def MakeNagFilename(): """Returns the filename for the nag file for this user.""" - user_homedir = os.path.expanduser("~/") + user_homedir = os.path.expanduser('~/') if not os.path.isdir(user_homedir): drive, unused_tail = os.path.splitdrive(os.__file__) if drive: - os.environ["HOMEDRIVE"] = drive + os.environ['HOMEDRIVE'] = drive - return os.path.expanduser("~/" + NAG_FILE) + return os.path.expanduser('~/' + NAG_FILE) def _ParseVersionFile(self): """Parse the local VERSION file. @@ -260,14 +315,14 @@ class UpdateCheck(object): """ version = self._ParseVersionFile() if version is None: - logging.error("Could not determine if the SDK supports the api_version " - "requested in app.yaml.") + logging.error('Could not determine if the SDK supports the api_version ' + 'requested in app.yaml.') return - if self.config.api_version not in version["api_versions"]: - logging.critical("The api_version specified in app.yaml (%s) is not " - "supported by this release of the SDK. The supported " - "api_versions are %s.", - self.config.api_version, version["api_versions"]) + if self.config.api_version not in version['api_versions']: + logging.critical('The api_version specified in app.yaml (%s) is not ' + 'supported by this release of the SDK. The supported ' + 'api_versions are %s.', + self.config.api_version, version['api_versions']) sys.exit(1) def CheckForUpdates(self): @@ -276,9 +331,9 @@ class UpdateCheck(object): Queries the server for the latest SDK version at the same time reporting the local SDK version. The server will respond with a yaml document containing the fields: - "release": The name of the release (e.g. 1.2). - "timestamp": The time the release was created (YYYY-MM-DD HH:MM AM/PM TZ). - "api_versions": A list of api_version strings (e.g. ['1', 'beta']). + 'release': The name of the release (e.g. 1.2). + 'timestamp': The time the release was created (YYYY-MM-DD HH:MM AM/PM TZ). + 'api_versions': A list of api_version strings (e.g. ['1', 'beta']). We will nag the user with increasing severity if: - There is a new release. @@ -288,42 +343,58 @@ class UpdateCheck(object): """ version = self._ParseVersionFile() if version is None: - logging.info("Skipping update check") + logging.info('Skipping update check') return - logging.info("Checking for updates to the SDK.") + logging.info('Checking for updates to the SDK.') try: - response = self.server.Send("/api/updatecheck", + response = self.server.Send('/api/updatecheck', timeout=UPDATE_CHECK_TIMEOUT, - release=version["release"], - timestamp=version["timestamp"], - api_versions=version["api_versions"]) + release=version['release'], + timestamp=version['timestamp'], + api_versions=version['api_versions']) except urllib2.URLError, e: - logging.info("Update check failed: %s", e) + logging.info('Update check failed: %s', e) return latest = yaml.safe_load(response) - if latest["release"] == version["release"]: - logging.info("The SDK is up to date.") + if version['release'] == latest['release']: + logging.info('The SDK is up to date.') return - api_versions = latest["api_versions"] + try: + this_release = _VersionList(version['release']) + except ValueError: + logging.warn('Could not parse this release version (%r)', + version['release']) + else: + try: + advertised_release = _VersionList(latest['release']) + except ValueError: + logging.warn('Could not parse advertised release version (%r)', + latest['release']) + else: + if this_release > advertised_release: + logging.info('This SDK release is newer than the advertised release.') + return + + api_versions = latest['api_versions'] if self.config.api_version not in api_versions: self._Nag( - "The api version you are using (%s) is obsolete! You should\n" - "upgrade your SDK and test that your code works with the new\n" - "api version." % self.config.api_version, + 'The api version you are using (%s) is obsolete! You should\n' + 'upgrade your SDK and test that your code works with the new\n' + 'api version.' % self.config.api_version, latest, version, force=True) return if self.config.api_version != api_versions[len(api_versions) - 1]: self._Nag( - "The api version you are using (%s) is deprecated. You should\n" - "upgrade your SDK to try the new functionality." % + 'The api version you are using (%s) is deprecated. You should\n' + 'upgrade your SDK to try the new functionality.' % self.config.api_version, latest, version) return - self._Nag("There is a new release of the SDK available.", + self._Nag('There is a new release of the SDK available.', latest, version) def _ParseNagFile(self): @@ -334,7 +405,7 @@ class UpdateCheck(object): """ nag_filename = UpdateCheck.MakeNagFilename() if self.isfile(nag_filename): - fh = self.open(nag_filename, "r") + fh = self.open(nag_filename, 'r') try: nag = NagFile.Load(fh) finally: @@ -353,13 +424,13 @@ class UpdateCheck(object): """ nagfilename = UpdateCheck.MakeNagFilename() try: - fh = self.open(nagfilename, "w") + fh = self.open(nagfilename, 'w') try: fh.write(nag.ToYAML()) finally: fh.close() except (OSError, IOError), e: - logging.error("Could not write nag file to %s. Error: %s", nagfilename, e) + logging.error('Could not write nag file to %s. Error: %s', nagfilename, e) def _Nag(self, msg, latest, version, force=False): """Prints a nag message and updates the nag file's timestamp. @@ -379,7 +450,7 @@ class UpdateCheck(object): if nag and not force: last_nag = datetime.datetime.fromtimestamp(nag.timestamp) if datetime.datetime.now() - last_nag < datetime.timedelta(weeks=1): - logging.debug("Skipping nag message") + logging.debug('Skipping nag message') return if nag is None: @@ -387,17 +458,17 @@ class UpdateCheck(object): nag.timestamp = time.time() self._WriteNagFile(nag) - print "****************************************************************" + print '****************************************************************' print msg - print "-----------" - print "Latest SDK:" + print '-----------' + print 'Latest SDK:' print yaml.dump(latest) - print "-----------" - print "Your SDK:" + print '-----------' + print 'Your SDK:' print yaml.dump(version) - print "-----------" - print "Please visit http://code.google.com/appengine for the latest SDK" - print "****************************************************************" + print '-----------' + print 'Please visit http://code.google.com/appengine for the latest SDK' + print '****************************************************************' def AllowedToCheckForUpdates(self, input_fn=raw_input): """Determines if the user wants to check for updates. @@ -423,16 +494,16 @@ class UpdateCheck(object): nag.timestamp = time.time() if nag.opt_in is None: - answer = input_fn("Allow dev_appserver to check for updates on startup? " - "(Y/n): ") + answer = input_fn('Allow dev_appserver to check for updates on startup? ' + '(Y/n): ') answer = answer.strip().lower() - if answer == "n" or answer == "no": - print ("dev_appserver will not check for updates on startup. To " - "change this setting, edit %s" % UpdateCheck.MakeNagFilename()) + if answer == 'n' or answer == 'no': + print ('dev_appserver will not check for updates on startup. To ' + 'change this setting, edit %s' % UpdateCheck.MakeNagFilename()) nag.opt_in = False else: - print ("dev_appserver will check for updates on startup. To change " - "this setting, edit %s" % UpdateCheck.MakeNagFilename()) + print ('dev_appserver will check for updates on startup. To change ' + 'this setting, edit %s' % UpdateCheck.MakeNagFilename()) nag.opt_in = True self._WriteNagFile(nag) return nag.opt_in @@ -456,8 +527,8 @@ class IndexDefinitionUpload(object): def DoUpload(self): """Uploads the index definitions.""" - StatusUpdate("Uploading index definitions.") - self.server.Send("/api/datastore/index/add", + StatusUpdate('Uploading index definitions.') + self.server.Send('/api/datastore/index/add', app_id=self.config.application, version=self.config.version, payload=self.definitions.ToYAML()) @@ -481,13 +552,38 @@ class CronEntryUpload(object): def DoUpload(self): """Uploads the cron entries.""" - StatusUpdate("Uploading cron entries.") - self.server.Send("/api/datastore/cron/update", + StatusUpdate('Uploading cron entries.') + self.server.Send('/api/datastore/cron/update', app_id=self.config.application, version=self.config.version, payload=self.cron.ToYAML()) +class QueueEntryUpload(object): + """Provides facilities to upload task queue entries to the hosting service.""" + + def __init__(self, server, config, queue): + """Creates a new QueueEntryUpload. + + Args: + server: The RPC server to use. Should be an instance of a subclass of + AbstractRpcServer + config: The AppInfoExternal object derived from the app.yaml file. + queue: The QueueInfoExternal object loaded from the queue.yaml file. + """ + self.server = server + self.config = config + self.queue = queue + + def DoUpload(self): + """Uploads the task queue entries.""" + StatusUpdate('Uploading task queue entries.') + self.server.Send('/api/queue/update', + app_id=self.config.application, + version=self.config.version, + payload=self.queue.ToYAML()) + + class IndexOperation(object): """Provide facilities for writing Index operation commands.""" @@ -516,8 +612,8 @@ class IndexOperation(object): present on the server but missing from the index.yaml file (indicating that these indexes should probably be vacuumed). """ - StatusUpdate("Fetching index definitions diff.") - response = self.server.Send("/api/datastore/index/diff", + StatusUpdate('Fetching index definitions diff.') + response = self.server.Send('/api/datastore/index/diff', app_id=self.config.application, payload=definitions.ToYAML()) return datastore_index.ParseMultipleIndexDefinitions(response) @@ -534,8 +630,8 @@ class IndexOperation(object): be normal behavior as there is a potential race condition between fetching the index-diff and sending deletion confirmation through. """ - StatusUpdate("Deleting selected index definitions.") - response = self.server.Send("/api/datastore/index/delete", + StatusUpdate('Deleting selected index definitions.') + response = self.server.Send('/api/datastore/index/delete', app_id=self.config.application, payload=definitions.ToYAML()) return datastore_index.ParseIndexDefinitions(response) @@ -581,24 +677,24 @@ class VacuumIndexesOperation(IndexOperation): True if user enters 'y' or 'a'. False if user enter 'n'. """ while True: - print "This index is no longer defined in your index.yaml file." + print 'This index is no longer defined in your index.yaml file.' print print index.ToYAML() print confirmation = self.confirmation_fn( - "Are you sure you want to delete this index? (N/y/a): ") + 'Are you sure you want to delete this index? (N/y/a): ') confirmation = confirmation.strip().lower() - if confirmation == "y": + if confirmation == 'y': return True - elif confirmation == "n" or not confirmation: + elif confirmation == 'n' or not confirmation: return False - elif confirmation == "a": + elif confirmation == 'a': self.force = True return True else: - print "Did not understand your response." + print 'Did not understand your response.' def DoVacuum(self, definitions): """Vacuum indexes in datastore. @@ -632,11 +728,11 @@ class VacuumIndexesOperation(IndexOperation): if not_deleted.indexes: not_deleted_count = len(not_deleted.indexes) if not_deleted_count == 1: - warning_message = ("An index was not deleted. Most likely this is " - "because it no longer exists.\n\n") + warning_message = ('An index was not deleted. Most likely this is ' + 'because it no longer exists.\n\n') else: - warning_message = ("%d indexes were not deleted. Most likely this " - "is because they no longer exist.\n\n" + warning_message = ('%d indexes were not deleted. Most likely this ' + 'is because they no longer exist.\n\n' % not_deleted_count) for index in not_deleted.indexes: warning_message += index.ToYAML() @@ -647,7 +743,7 @@ class LogsRequester(object): """Provide facilities to export request logs.""" def __init__(self, server, config, output_file, - num_days, append, severity, now): + num_days, append, severity, now, vhost, include_vhost): """Constructor. Args: @@ -659,6 +755,8 @@ class LogsRequester(object): append: True if appending to an existing file. severity: App log severity to request (0-4); None for no app logs. now: POSIX timestamp used for calculating valid dates for num_days. + vhost: The virtual host of log messages to get. None for all hosts. + include_vhost: If true, the virtual host is included in log messages. """ self.server = server self.config = config @@ -666,21 +764,23 @@ class LogsRequester(object): self.append = append self.num_days = num_days self.severity = severity - self.version_id = self.config.version + ".1" + self.vhost = vhost + self.include_vhost = include_vhost + self.version_id = self.config.version + '.1' self.sentinel = None - self.write_mode = "w" + self.write_mode = 'w' if self.append: self.sentinel = FindSentinel(self.output_file) - self.write_mode = "a" + self.write_mode = 'a' self.valid_dates = None if self.num_days: patterns = [] now = PacificTime(now) for i in xrange(self.num_days): then = time.gmtime(now - 24*3600 * i) - patterns.append(re.escape(time.strftime("%d/%m/%Y", then))) - patterns.append(re.escape(time.strftime("%d/%b/%Y", then))) - self.valid_dates = re.compile(r"[^[]+\[(" + "|".join(patterns) + r"):") + patterns.append(re.escape(time.strftime('%d/%m/%Y', then))) + patterns.append(re.escape(time.strftime('%d/%b/%Y', then))) + self.valid_dates = re.compile(r'[^[]+\[(' + '|'.join(patterns) + r'):') def DownloadLogs(self): """Download the requested logs. @@ -689,7 +789,7 @@ class LogsRequester(object): self.output_file, or to stdout if the filename is '-'. Multiple roundtrips to the server may be made. """ - StatusUpdate("Downloading request logs for %s %s." % + StatusUpdate('Downloading request logs for %s %s.' % (self.config.application, self.version_id)) tf = tempfile.TemporaryFile() offset = None @@ -700,16 +800,16 @@ class LogsRequester(object): if not offset: break except KeyboardInterrupt: - StatusUpdate("Keyboard interrupt; saving data downloaded so far.") + StatusUpdate('Keyboard interrupt; saving data downloaded so far.') break - StatusUpdate("Copying request logs to %r." % self.output_file) - if self.output_file == "-": + StatusUpdate('Copying request logs to %r.' % self.output_file) + if self.output_file == '-': of = sys.stdout else: try: of = open(self.output_file, self.write_mode) except IOError, err: - StatusUpdate("Can't write %r: %s." % (self.output_file, err)) + StatusUpdate('Can\'t write %r: %s.' % (self.output_file, err)) sys.exit(1) try: line_count = CopyReversedLines(tf, of) @@ -719,7 +819,7 @@ class LogsRequester(object): of.close() finally: tf.close() - StatusUpdate("Copied %d records." % line_count) + StatusUpdate('Copied %d records.' % line_count) def RequestLogLines(self, tf, offset): """Make a single roundtrip to the server. @@ -734,26 +834,30 @@ class LogsRequester(object): The offset string to be used for the next request, if another request should be issued; or None, if not. """ - logging.info("Request with offset %r.", offset) - kwds = {"app_id": self.config.application, - "version": self.version_id, - "limit": 100, + logging.info('Request with offset %r.', offset) + kwds = {'app_id': self.config.application, + 'version': self.version_id, + 'limit': 100, } if offset: - kwds["offset"] = offset + kwds['offset'] = offset if self.severity is not None: - kwds["severity"] = str(self.severity) - response = self.server.Send("/api/request_logs", payload=None, **kwds) - response = response.replace("\r", "\0") + kwds['severity'] = str(self.severity) + if self.vhost is not None: + kwds['vhost'] = str(self.vhost) + if self.include_vhost is not None: + kwds['include_vhost'] = str(self.include_vhost) + response = self.server.Send('/api/request_logs', payload=None, **kwds) + response = response.replace('\r', '\0') lines = response.splitlines() - logging.info("Received %d bytes, %d records.", len(response), len(lines)) + logging.info('Received %d bytes, %d records.', len(response), len(lines)) offset = None - if lines and lines[0].startswith("#"): - match = re.match(r"^#\s*next_offset=(\S+)\s*$", lines[0]) + if lines and lines[0].startswith('#'): + match = re.match(r'^#\s*next_offset=(\S+)\s*$', lines[0]) del lines[0] if match: offset = match.group(1) - if lines and lines[-1].startswith("#"): + if lines and lines[-1].startswith('#'): del lines[-1] valid_dates = self.valid_dates sentinel = self.sentinel @@ -763,10 +867,10 @@ class LogsRequester(object): for line in lines: if ((sentinel and line.startswith(sentinel) and - line[len_sentinel : len_sentinel+1] in ("", "\0")) or + line[len_sentinel : len_sentinel+1] in ('', '\0')) or (valid_dates and not valid_dates.match(line))): return None - tf.write(line + "\n") + tf.write(line + '\n') if not lines: return None return offset @@ -790,7 +894,7 @@ def PacificTime(now): header uses UTC), and the client's local time is irrelevant. Args: - A posix timestamp giving current UTC time. + now: A posix timestamp giving current UTC time. Returns: A pseudo-posix timestamp giving current Pacific time. Passing @@ -830,9 +934,9 @@ def CopyReversedLines(instream, outstream, blocksize=2**16): r"""Copy lines from input stream to output stream in reverse order. As a special feature, null bytes in the input are turned into - newlines followed by tabs in the output, but these "sub-lines" + newlines followed by tabs in the output, but these 'sub-lines' separated by null bytes are not reversed. E.g. If the input is - "A\0B\nC\0D\n", the output is "C\n\tD\nA\n\tB\n". + 'A\0B\nC\0D\n', the output is 'C\n\tD\nA\n\tB\n'. Args: instream: A seekable stream open for reading in binary mode. @@ -845,20 +949,20 @@ def CopyReversedLines(instream, outstream, blocksize=2**16): line_count = 0 instream.seek(0, 2) last_block = instream.tell() // blocksize - spillover = "" + spillover = '' for iblock in xrange(last_block + 1, -1, -1): instream.seek(iblock * blocksize) data = instream.read(blocksize) lines = data.splitlines(True) - lines[-1:] = "".join(lines[-1:] + [spillover]).splitlines(True) - if lines and not lines[-1].endswith("\n"): - lines[-1] += "\n" + lines[-1:] = ''.join(lines[-1:] + [spillover]).splitlines(True) + if lines and not lines[-1].endswith('\n'): + lines[-1] += '\n' lines.reverse() if lines and iblock > 0: spillover = lines.pop() if lines: line_count += len(lines) - data = "".join(lines).replace("\0", "\n\t") + data = ''.join(lines).replace('\0', '\n\t') outstream.write(data) return line_count @@ -876,13 +980,13 @@ def FindSentinel(filename, blocksize=2**16): couldn't be opened or no such line could be found by inspecting the last 'blocksize' bytes of the file. """ - if filename == "-": - StatusUpdate("Can't combine --append with output to stdout.") + if filename == '-': + StatusUpdate('Can\'t combine --append with output to stdout.') sys.exit(2) try: - fp = open(filename, "rb") + fp = open(filename, 'rb') except IOError, err: - StatusUpdate("Append mode disabled: can't read %r: %s." % (filename, err)) + StatusUpdate('Append mode disabled: can\'t read %r: %s.' % (filename, err)) return None try: fp.seek(0, 2) @@ -891,17 +995,160 @@ def FindSentinel(filename, blocksize=2**16): del lines[:1] sentinel = None for line in lines: - if not line.startswith("\t"): + if not line.startswith('\t'): sentinel = line if not sentinel: - StatusUpdate("Append mode disabled: can't find sentinel in %r." % + StatusUpdate('Append mode disabled: can\'t find sentinel in %r.' % filename) return None - return sentinel.rstrip("\n") + return sentinel.rstrip('\n') finally: fp.close() +class UploadBatcher(object): + """Helper to batch file uploads.""" + + def __init__(self, what, app_id, version, server): + """Constructor. + + Args: + what: Either 'file' or 'blob' indicating what kind of objects + this batcher uploads. Used in messages and URLs. + app_id: The application ID. + version: The application version string. + server: The RPC server. + """ + assert what in ('file', 'blob'), repr(what) + self.what = what + self.app_id = app_id + self.version = version + self.server = server + self.single_url = '/api/appversion/add' + what + self.batch_url = self.single_url + 's' + self.batching = True + self.batch = [] + self.batch_size = 0 + + def SendBatch(self): + """Send the current batch on its way. + + If successful, resets self.batch and self.batch_size. + + Raises: + HTTPError with code=404 if the server doesn't support batching. + """ + boundary = 'boundary' + parts = [] + for path, payload, mime_type in self.batch: + while boundary in payload: + boundary += '%04x' % random.randint(0, 0xffff) + assert len(boundary) < 80, 'Unexpected error, please try again.' + part = '\n'.join(['', + 'X-Appcfg-File: %s' % urllib.quote(path), + 'X-Appcfg-Hash: %s' % _Hash(payload), + 'Content-Type: %s' % mime_type, + 'Content-Length: %d' % len(payload), + 'Content-Transfer-Encoding: 8bit', + '', + payload, + ]) + parts.append(part) + parts.insert(0, + 'MIME-Version: 1.0\n' + 'Content-Type: multipart/mixed; boundary="%s"\n' + '\n' + 'This is a message with multiple parts in MIME format.' % + boundary) + parts.append('--\n') + delimiter = '\n--%s' % boundary + payload = delimiter.join(parts) + logging.info('Uploading batch of %d %ss to %s with boundary="%s".', + len(self.batch), self.what, self.batch_url, boundary) + self.server.Send(self.batch_url, + payload=payload, + content_type='message/rfc822', + app_id=self.app_id, + version=self.version) + self.batch = [] + self.batch_size = 0 + + def SendSingleFile(self, path, payload, mime_type): + """Send a single file on its way.""" + logging.info('Uploading %s %s (%s bytes, type=%s) to %s.', + self.what, path, len(payload), mime_type, self.single_url) + self.server.Send(self.single_url, + payload=payload, + content_type=mime_type, + path=path, + app_id=self.app_id, + version=self.version) + + def Flush(self): + """Flush the current batch. + + This first attempts to send the batch as a single request; if that + fails because the server doesn't support batching, the files are + sent one by one, and self.batching is reset to False. + + At the end, self.batch and self.batch_size are reset. + """ + if not self.batch: + return + try: + self.SendBatch() + except urllib2.HTTPError, err: + if err.code != 404: + raise + + logging.info('Old server detected; turning off %s batching.', self.what) + self.batching = False + + for path, payload, mime_type in self.batch: + self.SendSingleFile(path, payload, mime_type) + + self.batch = [] + self.batch_size = 0 + + def AddToBatch(self, path, payload, mime_type): + """Batch a file, possibly flushing first, or perhaps upload it directly. + + Args: + path: The name of the file. + payload: The contents of the file. + mime_type: The MIME Content-type of the file, or None. + + If mime_type is None, application/octet-stream is substituted. + """ + if not mime_type: + mime_type = 'application/octet-stream' + size = len(payload) + if size <= MAX_BATCH_FILE_SIZE: + if (len(self.batch) >= MAX_BATCH_COUNT or + self.batch_size + size > MAX_BATCH_SIZE): + self.Flush() + if self.batching: + logging.info('Adding %s %s (%s bytes, type=%s) to batch.', + self.what, path, size, mime_type) + self.batch.append((path, payload, mime_type)) + self.batch_size += size + BATCH_OVERHEAD + return + self.SendSingleFile(path, payload, mime_type) + + +def _Hash(content): + """Compute the hash of the content. + + Args: + content: The data to hash as a string. + + Returns: + The string representation of the hash. + """ + h = sha.new(content).hexdigest() + return '%s_%s_%s_%s_%s' % (h[0:8], h[8:16], h[16:24], h[24:32], h[32:40]) + + class AppVersionUpload(object): """Provides facilities to upload a new appversion to the hosting service. @@ -914,6 +1161,7 @@ class AppVersionUpload(object): hash of the file contents. in_transaction: True iff a transaction with the server has started. An AppVersionUpload can do only one transaction at a time. + deployed: True iff the Deploy method has been called. """ def __init__(self, server, config): @@ -931,18 +1179,12 @@ class AppVersionUpload(object): self.version = self.config.version self.files = {} self.in_transaction = False - - def _Hash(self, content): - """Compute the hash of the content. - - Args: - content: The data to hash as a string. - - Returns: - The string representation of the hash. - """ - h = sha.new(content).hexdigest() - return "%s_%s_%s_%s_%s" % (h[0:8], h[8:16], h[16:24], h[24:32], h[32:40]) + self.deployed = False + self.batching = True + self.file_batcher = UploadBatcher('file', self.app_id, self.version, + self.server) + self.blob_batcher = UploadBatcher('blob', self.app_id, self.version, + self.server) def AddFile(self, path, file_handle): """Adds the provided file to the list to be pushed to the server. @@ -951,7 +1193,7 @@ class AppVersionUpload(object): path: The path the file should be uploaded as. file_handle: A stream containing data to upload. """ - assert not self.in_transaction, "Already in a transaction." + assert not self.in_transaction, 'Already in a transaction.' assert file_handle is not None reason = appinfo.ValidFilename(path) @@ -960,7 +1202,7 @@ class AppVersionUpload(object): return pos = file_handle.tell() - content_hash = self._Hash(file_handle.read()) + content_hash = _Hash(file_handle.read()) file_handle.seek(pos, 0) self.files[path] = content_hash @@ -974,10 +1216,10 @@ class AppVersionUpload(object): A list of pathnames for files that should be uploaded using UploadFile() before Commit() can be called. """ - assert not self.in_transaction, "Already in a transaction." + assert not self.in_transaction, 'Already in a transaction.' - StatusUpdate("Initiating update.") - self.server.Send("/api/appversion/create", app_id=self.app_id, + StatusUpdate('Initiating update.') + self.server.Send('/api/appversion/create', app_id=self.app_id, version=self.version, payload=self.config.ToYAML()) self.in_transaction = True @@ -1003,11 +1245,11 @@ class AppVersionUpload(object): if not files: return - StatusUpdate("Cloning %d %s file%s." % - (len(files), file_type, len(files) != 1 and "s" or "")) + StatusUpdate('Cloning %d %s file%s.' % + (len(files), file_type, len(files) != 1 and 's' or '')) for i in xrange(0, len(files), MAX_FILES_TO_CLONE): if i > 0 and i % MAX_FILES_TO_CLONE == 0: - StatusUpdate("Cloned %d files." % i) + StatusUpdate('Cloned %d files.' % i) chunk = files[i:min(len(files), i + MAX_FILES_TO_CLONE)] result = self.server.Send(url, @@ -1017,10 +1259,10 @@ class AppVersionUpload(object): files_to_upload.update(dict( (f, self.files[f]) for f in result.split(LIST_DELIMITER))) - CloneFiles("/api/appversion/cloneblobs", blobs_to_clone, "static") - CloneFiles("/api/appversion/clonefiles", files_to_clone, "application") + CloneFiles('/api/appversion/cloneblobs', blobs_to_clone, 'static') + CloneFiles('/api/appversion/clonefiles', files_to_clone, 'application') - logging.info("Files to upload: " + str(files_to_upload)) + logging.debug('Files to upload: %s', files_to_upload) self.files = files_to_upload return sorted(files_to_upload.iterkeys()) @@ -1038,21 +1280,18 @@ class AppVersionUpload(object): Raises: KeyError: The provided file is not amongst those to be uploaded. """ - assert self.in_transaction, "Begin() must be called before UploadFile()." + assert self.in_transaction, 'Begin() must be called before UploadFile().' if path not in self.files: - raise KeyError("File '%s' is not in the list of files to be uploaded." + raise KeyError('File \'%s\' is not in the list of files to be uploaded.' % path) del self.files[path] mime_type = GetMimeTypeIfStaticFile(self.config, path) - if mime_type is not None: - self.server.Send("/api/appversion/addblob", app_id=self.app_id, - version=self.version, path=path, content_type=mime_type, - payload=file_handle.read()) + payload = file_handle.read() + if mime_type is None: + self.file_batcher.AddToBatch(path, payload, mime_type) else: - self.server.Send("/api/appversion/addfile", app_id=self.app_id, - version=self.version, path=path, - payload=file_handle.read()) + self.blob_batcher.AddToBatch(path, payload, mime_type) def Commit(self): """Commits the transaction, making the new app version available. @@ -1060,24 +1299,82 @@ class AppVersionUpload(object): All the files returned by Begin() must have been uploaded with UploadFile() before Commit() can be called. + This tries the new 'deploy' method; if that fails it uses the old 'commit'. + + Raises: + Exception: Some required files were not uploaded. + """ + assert self.in_transaction, 'Begin() must be called before Commit().' + if self.files: + raise Exception('Not all required files have been uploaded.') + + try: + self.Deploy() + if not RetryWithBackoff(1, 2, 8, self.IsReady): + logging.warning('Version still not ready to serve, aborting.') + raise Exception('Version not ready.') + self.StartServing() + except urllib2.HTTPError, e: + if e.code != 404: + raise + StatusUpdate('Closing update.') + self.server.Send('/api/appversion/commit', app_id=self.app_id, + version=self.version) + self.in_transaction = False + + def Deploy(self): + """Deploys the new app version but does not make it default. + + All the files returned by Begin() must have been uploaded with UploadFile() + before Deploy() can be called. + Raises: Exception: Some required files were not uploaded. """ - assert self.in_transaction, "Begin() must be called before Commit()." + assert self.in_transaction, 'Begin() must be called before Deploy().' if self.files: - raise Exception("Not all required files have been uploaded.") + raise Exception('Not all required files have been uploaded.') - StatusUpdate("Closing update.") - self.server.Send("/api/appversion/commit", app_id=self.app_id, + StatusUpdate('Deploying new version.') + self.server.Send('/api/appversion/deploy', app_id=self.app_id, version=self.version) + self.deployed = True + + def IsReady(self): + """Check if the new app version is ready to serve traffic. + + Raises: + Exception: Deploy has not yet been called. + + Returns: + True if the server returned the app is ready to serve. + """ + assert self.deployed, 'Deploy() must be called before IsReady().' + + StatusUpdate('Checking if new version is ready to serve.') + result = self.server.Send('/api/appversion/isready', app_id=self.app_id, + version=self.version) + return result == '1' + + def StartServing(self): + """Start serving with the newly created version. + + Raises: + Exception: Deploy has not yet been called. + """ + assert self.deployed, 'Deploy() must be called before IsReady().' + + StatusUpdate('Closing update: new version is ready to start serving.') + self.server.Send('/api/appversion/startserving', + app_id=self.app_id, version=self.version) self.in_transaction = False def Rollback(self): """Rolls back the transaction if one is in progress.""" if not self.in_transaction: return - StatusUpdate("Rolling back the update.") - self.server.Send("/api/appversion/rollback", app_id=self.app_id, + StatusUpdate('Rolling back the update.') + self.server.Send('/api/appversion/rollback', app_id=self.app_id, version=self.version) self.in_transaction = False self.files = {} @@ -1090,47 +1387,46 @@ class AppVersionUpload(object): max_size: The maximum size file to upload. openfunc: A function that takes a path and returns a file-like object. """ - logging.info("Reading app configuration.") + logging.info('Reading app configuration.') - path = "" + path = '' try: - StatusUpdate("Scanning files on local disk.") + StatusUpdate('Scanning files on local disk.') num_files = 0 for path in paths: file_handle = openfunc(path) try: if self.config.skip_files.match(path): - logging.info("Ignoring file '%s': File matches ignore regex.", + logging.info('Ignoring file \'%s\': File matches ignore regex.', path) else: file_length = GetFileLength(file_handle) if file_length > max_size: - logging.error("Ignoring file '%s': Too long " - "(max %d bytes, file is %d bytes)", + logging.error('Ignoring file \'%s\': Too long ' + '(max %d bytes, file is %d bytes)', path, max_size, file_length) else: - logging.info("Processing file '%s'", path) + logging.info('Processing file \'%s\'', path) self.AddFile(path, file_handle) finally: file_handle.close() num_files += 1 if num_files % 500 == 0: - StatusUpdate("Scanned %d files." % num_files) + StatusUpdate('Scanned %d files.' % num_files) except KeyboardInterrupt: - logging.info("User interrupted. Aborting.") + logging.info('User interrupted. Aborting.') raise except EnvironmentError, e: - logging.error("An error occurred processing file '%s': %s. Aborting.", + logging.error('An error occurred processing file \'%s\': %s. Aborting.', path, e) raise try: missing_files = self.Begin() if missing_files: - StatusUpdate("Uploading %d files." % len(missing_files)) + StatusUpdate('Uploading %d files and blobs.' % len(missing_files)) num_files = 0 for missing_file in missing_files: - logging.info("Uploading file '%s'" % missing_file) file_handle = openfunc(missing_file) try: self.UploadFile(missing_file, file_handle) @@ -1138,19 +1434,28 @@ class AppVersionUpload(object): file_handle.close() num_files += 1 if num_files % 500 == 0: - StatusUpdate("Uploaded %d files." % num_files) + StatusUpdate('Processed %d out of %s.' % + (num_files, len(missing_files))) + self.file_batcher.Flush() + self.blob_batcher.Flush() + StatusUpdate('Uploaded %d files and blobs' % num_files) self.Commit() + except KeyboardInterrupt: - logging.info("User interrupted. Aborting.") + logging.info('User interrupted. Aborting.') + self.Rollback() + raise + except urllib2.HTTPError, err: + logging.info('HTTP Error (%s)', err) self.Rollback() raise except: - logging.error("An unexpected error occurred. Aborting.") + logging.exception('An unexpected error occurred. Aborting.') self.Rollback() raise - logging.info("Done!") + logging.info('Done!') def FileIterator(base, separator=os.path.sep): @@ -1163,15 +1468,15 @@ def FileIterator(base, separator=os.path.sep): Yields: Paths of files found, relative to base. """ - dirs = [""] + dirs = [''] while dirs: current_dir = dirs.pop() for entry in os.listdir(os.path.join(base, current_dir)): name = os.path.join(current_dir, entry) fullname = os.path.join(base, name) if os.path.isfile(fullname): - if separator == "\\": - name = name.replace("\\", "/") + if separator == '\\': + name = name.replace('\\', '/') yield name elif os.path.isdir(fullname): dirs.append(name) @@ -1210,38 +1515,38 @@ def GetUserAgent(get_version=GetVersionObject, Returns: String containing the 'user-agent' header value, which includes the SDK version, the platform information, and the version of Python; - e.g., "appcfg_py/1.0.1 Darwin/9.2.0 Python/2.5.2". + e.g., 'appcfg_py/1.0.1 Darwin/9.2.0 Python/2.5.2'. """ product_tokens = [] - sdk_name = os.environ.get("APPCFG_SDK_NAME") + sdk_name = os.environ.get('APPCFG_SDK_NAME') if sdk_name: product_tokens.append(sdk_name) else: version = get_version() if version is None: - release = "unknown" + release = 'unknown' else: - release = version["release"] + release = version['release'] - product_tokens.append("appcfg_py/%s" % release) + product_tokens.append('appcfg_py/%s' % release) product_tokens.append(get_platform()) - python_version = ".".join(str(i) for i in sys.version_info) - product_tokens.append("Python/%s" % python_version) + python_version = '.'.join(str(i) for i in sys.version_info) + product_tokens.append('Python/%s' % python_version) - return " ".join(product_tokens) + return ' '.join(product_tokens) def GetSourceName(get_version=GetVersionObject): """Gets the name of this source version.""" version = get_version() if version is None: - release = "unknown" + release = 'unknown' else: - release = version["release"] - return "Google-appcfg-%s" % (release,) + release = version['release'] + return 'Google-appcfg-%s' % (release,) class AppCfgApp(object): @@ -1272,7 +1577,8 @@ class AppCfgApp(object): rpc_server_class=appengine_rpc.HttpRpcServer, raw_input_fn=raw_input, password_input_fn=getpass.getpass, - error_fh=sys.stderr): + error_fh=sys.stderr, + update_check_class=UpdateCheck): """Initializer. Parses the cmdline and selects the Action to use. Initializes all of the attributes described in the class docstring. @@ -1285,6 +1591,7 @@ class AppCfgApp(object): raw_input_fn: Function used for getting user email. password_input_fn: Function used for getting user password. error_fh: Unexpected HTTPErrors are printed to this file handle. + update_check_class: UpdateCheck class (can be replaced for testing). """ self.parser_class = parser_class self.argv = argv @@ -1292,6 +1599,7 @@ class AppCfgApp(object): self.raw_input_fn = raw_input_fn self.password_input_fn = password_input_fn self.error_fh = error_fh + self.update_check_class = update_check_class self.parser = self._GetOptionParser() for action in self.actions.itervalues(): @@ -1302,7 +1610,7 @@ class AppCfgApp(object): if len(self.args) < 1: self._PrintHelpAndExit() if self.args[0] not in self.actions: - self.parser.error("Unknown action '%s'\n%s" % + self.parser.error('Unknown action \'%s\'\n%s' % (self.args[0], self.parser.get_description())) action_name = self.args.pop(0) self.action = self.actions[action_name] @@ -1324,17 +1632,20 @@ class AppCfgApp(object): """Executes the requested action. Catches any HTTPErrors raised by the action and prints them to stderr. + + Returns: + 1 on error, 0 if successful. """ try: self.action(self) except urllib2.HTTPError, e: body = e.read() - print >>self.error_fh, ("Error %d: --- begin server output ---\n" - "%s\n--- end server output ---" % - (e.code, body.rstrip("\n"))) + print >>self.error_fh, ('Error %d: --- begin server output ---\n' + '%s\n--- end server output ---' % + (e.code, body.rstrip('\n'))) return 1 except yaml_errors.EventListenerError, e: - print >>self.error_fh, ("Error parsing yaml file:\n%s" % e) + print >>self.error_fh, ('Error parsing yaml file:\n%s' % e) return 1 return 0 @@ -1342,9 +1653,9 @@ class AppCfgApp(object): """Returns a formatted string containing the short_descs for all actions.""" action_names = self.actions.keys() action_names.sort() - desc = "" + desc = '' for action_name in action_names: - desc += " %s: %s\n" % (action_name, self.actions[action_name].short_desc) + desc += ' %s: %s\n' % (action_name, self.actions[action_name].short_desc) return desc def _GetOptionParser(self): @@ -1359,40 +1670,48 @@ class AppCfgApp(object): def format_description(self, description): """Very simple formatter.""" - return description + "\n" + return description + '\n' desc = self._GetActionDescriptions() - desc = ("Action must be one of:\n%s" - "Use 'help <action>' for a detailed description.") % desc + desc = ('Action must be one of:\n%s' + 'Use \'help <action>\' for a detailed description.') % desc - parser = self.parser_class(usage="%prog [options] <action>", + parser = self.parser_class(usage='%prog [options] <action>', description=desc, formatter=Formatter(), - conflict_handler="resolve") - parser.add_option("-h", "--help", action="store_true", - dest="help", help="Show the help message and exit.") - parser.add_option("-q", "--quiet", action="store_const", const=0, - dest="verbose", help="Print errors only.") - parser.add_option("-v", "--verbose", action="store_const", const=2, - dest="verbose", default=1, - help="Print info level logs.") - parser.add_option("--noisy", action="store_const", const=3, - dest="verbose", help="Print all logs.") - parser.add_option("-s", "--server", action="store", dest="server", - default="appengine.google.com", - metavar="SERVER", help="The server to connect to.") - parser.add_option("-e", "--email", action="store", dest="email", - metavar="EMAIL", default=None, - help="The username to use. Will prompt if omitted.") - parser.add_option("-H", "--host", action="store", dest="host", - metavar="HOST", default=None, - help="Overrides the Host header sent with all RPCs.") - parser.add_option("--no_cookies", action="store_false", - dest="save_cookies", default=True, - help="Do not save authentication cookies to local disk.") - parser.add_option("--passin", action="store_true", - dest="passin", default=False, - help="Read the login password from stdin.") + conflict_handler='resolve') + parser.add_option('-h', '--help', action='store_true', + dest='help', help='Show the help message and exit.') + parser.add_option('-q', '--quiet', action='store_const', const=0, + dest='verbose', help='Print errors only.') + parser.add_option('-v', '--verbose', action='store_const', const=2, + dest='verbose', default=1, + help='Print info level logs.') + parser.add_option('--noisy', action='store_const', const=3, + dest='verbose', help='Print all logs.') + parser.add_option('-s', '--server', action='store', dest='server', + default='appengine.google.com', + metavar='SERVER', help='The server to connect to.') + parser.add_option('--secure', action='store_true', dest='secure', + default=True, help=optparse.SUPPRESS_HELP) + parser.add_option('--insecure', action='store_false', dest='secure', + help='Use HTTP when communicating with the server.') + parser.add_option('-e', '--email', action='store', dest='email', + metavar='EMAIL', default=None, + help='The username to use. Will prompt if omitted.') + parser.add_option('-H', '--host', action='store', dest='host', + metavar='HOST', default=None, + help='Overrides the Host header sent with all RPCs.') + parser.add_option('--no_cookies', action='store_false', + dest='save_cookies', default=True, + help='Do not save authentication cookies to local disk.') + parser.add_option('--passin', action='store_true', + dest='passin', default=False, + help='Read the login password from stdin.') + parser.add_option('-A', '--application', action='store', dest='app_id', + help='Override application from app.yaml file.') + parser.add_option('-V', '--version', action='store', dest='version', + help='Override (major) version from app.yaml file.') return parser def _MakeSpecificParser(self, action): @@ -1408,7 +1727,7 @@ class AppCfgApp(object): """ parser = self._GetOptionParser() parser.set_usage(action.usage) - parser.set_description("%s\n%s" % (action.short_desc, action.long_desc)) + parser.set_description('%s\n%s' % (action.short_desc, action.long_desc)) action.options(self, parser) options, unused_args = parser.parse_args(self.argv[1:]) return parser, options @@ -1433,9 +1752,9 @@ class AppCfgApp(object): """Prompts the user for a username and password.""" email = self.options.email if email is None: - email = self.raw_input_fn("Email: ") + email = self.raw_input_fn('Email: ') - password_prompt = "Password for %s: " % email + password_prompt = 'Password for %s: ' % email if self.options.passin: password = self.raw_input_fn(password_prompt) else: @@ -1443,18 +1762,22 @@ class AppCfgApp(object): return (email, password) - if self.options.host and self.options.host == "localhost": + StatusUpdate('Server: %s.' % self.options.server) + + if self.options.host and self.options.host == 'localhost': email = self.options.email if email is None: - email = "test@example.com" - logging.info("Using debug user %s. Override with --email" % email) + email = 'test@example.com' + logging.info('Using debug user %s. Override with --email' % email) server = self.rpc_server_class( self.options.server, - lambda: (email, "password"), + lambda: (email, 'password'), GetUserAgent(), GetSourceName(), host_override=self.options.host, - save_cookies=self.options.save_cookies) + save_cookies=self.options.save_cookies, + + secure=False) server.authenticated = True return server @@ -1468,7 +1791,8 @@ class AppCfgApp(object): host_override=self.options.host, save_cookies=self.options.save_cookies, auth_tries=auth_tries, - account_type="HOSTED_OR_GOOGLE") + account_type='HOSTED_OR_GOOGLE', + secure=self.options.secure) def _FindYaml(self, basepath, file_name): """Find yaml files in application directory. @@ -1481,9 +1805,9 @@ class AppCfgApp(object): Path to located yaml file if one exists, else None. """ if not os.path.isdir(basepath): - self.parser.error("Not a directory: %s" % basepath) + self.parser.error('Not a directory: %s' % basepath) - for yaml_file in (file_name + ".yaml", file_name + ".yml"): + for yaml_file in (file_name + '.yaml', file_name + '.yml'): yaml_path = os.path.join(basepath, yaml_file) if os.path.isfile(yaml_path): return yaml_path @@ -1499,42 +1823,65 @@ class AppCfgApp(object): Returns: An AppInfoExternal object. """ - appyaml_filename = self._FindYaml(basepath, "app") + appyaml_filename = self._FindYaml(basepath, 'app') if appyaml_filename is None: - self.parser.error("Directory does not contain an app.yaml " - "configuration file.") - - from string import Template - appyaml_content = open(appyaml_filename, "r").read() - appyaml_template = Template(appyaml_content) - appyaml_config = appyaml_template.substitute(os.environ) - fh = cStringIO.StringIO(appyaml_config) + self.parser.error('Directory does not contain an app.yaml ' + 'configuration file.') + fh = open(appyaml_filename, 'r') try: appyaml = appinfo.LoadSingleAppInfo(fh) finally: fh.close() + orig_application = appyaml.application + orig_version = appyaml.version + if self.options.app_id: + appyaml.application = self.options.app_id + if self.options.version: + appyaml.version = self.options.version + msg = 'Application: %s' % appyaml.application + if appyaml.application != orig_application: + msg += ' (was: %s)' % orig_application + msg += '; version: %s' % appyaml.version + if appyaml.version != orig_version: + msg += ' (was: %s)' % orig_version + msg += '.' + StatusUpdate(msg) return appyaml - def _ParseIndexYaml(self, basepath): - """Parses the index.yaml file. + def _ParseYamlFile(self, basepath, basename, parser): + """Parses the a yaml file. Args: basepath: the directory of the application. + basename: the base name of the file (with the '.yaml' stripped off). + parser: the function or method used to parse the file. Returns: A single parsed yaml file or None if the file does not exist. """ - file_name = self._FindYaml(basepath, "index") + file_name = self._FindYaml(basepath, basename) if file_name is not None: - fh = open(file_name, "r") + fh = open(file_name, 'r') try: - index_defs = datastore_index.ParseIndexDefinitions(fh) + defns = parser(fh) finally: fh.close() - return index_defs + return defns return None + def _ParseIndexYaml(self, basepath): + """Parses the index.yaml file. + + Args: + basepath: the directory of the application. + + Returns: + A single parsed yaml file or None if the file does not exist. + """ + return self._ParseYamlFile(basepath, 'index', + datastore_index.ParseIndexDefinitions) + def _ParseCronYaml(self, basepath): """Parses the cron.yaml file. @@ -1542,17 +1889,20 @@ class AppCfgApp(object): basepath: the directory of the application. Returns: - A CronInfoExternal object. + A CronInfoExternal object or None if the file does not exist. """ - file_name = self._FindYaml(basepath, "cron") - if file_name is not None: - fh = open(file_name, "r") - try: - cron_info = croninfo.LoadSingleCron(fh) - finally: - fh.close() - return cron_info - return None + return self._ParseYamlFile(basepath, 'cron', croninfo.LoadSingleCron) + + def _ParseQueueYaml(self, basepath): + """Parses the queue.yaml file. + + Args: + basepath: the directory of the application. + + Returns: + A CronInfoExternal object or None if the file does not exist. + """ + return self._ParseYamlFile(basepath, 'queue', queueinfo.LoadSingleQueue) def Help(self): """Prints help for a specific action. @@ -1561,7 +1911,7 @@ class AppCfgApp(object): Exits the program after printing the help message. """ if len(self.args) != 1 or self.args[0] not in self.actions: - self.parser.error("Expected a single action argument. Must be one of:\n" + + self.parser.error('Expected a single action argument. Must be one of:\n' + self._GetActionDescriptions()) action = self.actions[self.args[0]] @@ -1571,18 +1921,18 @@ class AppCfgApp(object): def Update(self): """Updates and deploys a new appversion.""" if len(self.args) != 1: - self.parser.error("Expected a single <directory> argument.") + self.parser.error('Expected a single <directory> argument.') basepath = self.args[0] appyaml = self._ParseAppYaml(basepath) rpc_server = self._GetRpcServer() - updatecheck = UpdateCheck(rpc_server, appyaml) + updatecheck = self.update_check_class(rpc_server, appyaml) updatecheck.CheckForUpdates() appversion = AppVersionUpload(rpc_server, appyaml) appversion.DoUpload(FileIterator(basepath), self.options.max_size, - lambda path: open(os.path.join(basepath, path), "rb")) + lambda path: open(os.path.join(basepath, path), 'rb')) index_defs = self._ParseIndexYaml(basepath) if index_defs: @@ -1590,32 +1940,37 @@ class AppCfgApp(object): try: index_upload.DoUpload() except urllib2.HTTPError, e: - StatusUpdate("Error %d: --- begin server output ---\n" - "%s\n--- end server output ---" % - (e.code, e.read().rstrip("\n"))) + StatusUpdate('Error %d: --- begin server output ---\n' + '%s\n--- end server output ---' % + (e.code, e.read().rstrip('\n'))) print >> self.error_fh, ( - "Your app was updated, but there was an error updating your " - "indexes. Please retry later with appcfg.py update_indexes.") + 'Your app was updated, but there was an error updating your ' + 'indexes. Please retry later with appcfg.py update_indexes.') cron_entries = self._ParseCronYaml(basepath) if cron_entries: cron_upload = CronEntryUpload(rpc_server, appyaml, cron_entries) cron_upload.DoUpload() + queue_entries = self._ParseQueueYaml(basepath) + if queue_entries: + queue_upload = QueueEntryUpload(rpc_server, appyaml, queue_entries) + queue_upload.DoUpload() + def _UpdateOptions(self, parser): """Adds update-specific options to 'parser'. Args: parser: An instance of OptionsParser. """ - parser.add_option("-S", "--max_size", type="int", dest="max_size", - default=1048576, metavar="SIZE", - help="Maximum size of a file to upload.") + parser.add_option('-S', '--max_size', type='int', dest='max_size', + default=10485760, metavar='SIZE', + help='Maximum size of a file to upload.') def VacuumIndexes(self): """Deletes unused indexes.""" if len(self.args) != 1: - self.parser.error("Expected a single <directory> argument.") + self.parser.error('Expected a single <directory> argument.') basepath = self.args[0] config = self._ParseAppYaml(basepath) @@ -1636,14 +1991,14 @@ class AppCfgApp(object): Args: parser: An instance of OptionsParser. """ - parser.add_option("-f", "--force", action="store_true", dest="force_delete", + parser.add_option('-f', '--force', action='store_true', dest='force_delete', default=False, - help="Force deletion without being prompted.") + help='Force deletion without being prompted.') def UpdateCron(self): """Updates any new or changed cron definitions.""" if len(self.args) != 1: - self.parser.error("Expected a single <directory> argument.") + self.parser.error('Expected a single <directory> argument.') basepath = self.args[0] appyaml = self._ParseAppYaml(basepath) @@ -1657,7 +2012,7 @@ class AppCfgApp(object): def UpdateIndexes(self): """Updates indexes.""" if len(self.args) != 1: - self.parser.error("Expected a single <directory> argument.") + self.parser.error('Expected a single <directory> argument.') basepath = self.args[0] appyaml = self._ParseAppYaml(basepath) @@ -1668,10 +2023,24 @@ class AppCfgApp(object): index_upload = IndexDefinitionUpload(rpc_server, appyaml, index_defs) index_upload.DoUpload() + def UpdateQueues(self): + """Updates any new or changed task queue definitions.""" + if len(self.args) != 1: + self.parser.error('Expected a single <directory> argument.') + + basepath = self.args[0] + appyaml = self._ParseAppYaml(basepath) + rpc_server = self._GetRpcServer() + + queue_entries = self._ParseQueueYaml(basepath) + if queue_entries: + queue_upload = QueueEntryUpload(rpc_server, appyaml, queue_entries) + queue_upload.DoUpload() + def Rollback(self): """Does a rollback of any existing transaction for this app version.""" if len(self.args) != 1: - self.parser.error("Expected a single <directory> argument.") + self.parser.error('Expected a single <directory> argument.') basepath = self.args[0] appyaml = self._ParseAppYaml(basepath) @@ -1684,14 +2053,20 @@ class AppCfgApp(object): """Write request logs to a file.""" if len(self.args) != 2: self.parser.error( - "Expected a <directory> argument and an <output_file> argument.") + 'Expected a <directory> argument and an <output_file> argument.') if (self.options.severity is not None and not 0 <= self.options.severity <= MAX_LOG_LEVEL): self.parser.error( - "Severity range is 0 (DEBUG) through %s (CRITICAL)." % MAX_LOG_LEVEL) + 'Severity range is 0 (DEBUG) through %s (CRITICAL).' % MAX_LOG_LEVEL) if self.options.num_days is None: self.options.num_days = int(not self.options.append) + + try: + end_date = self._ParseEndDate(self.options.end_date) + except ValueError: + self.parser.error('End date must be in the format YYYY-MM-DD.') + basepath = self.args[0] appyaml = self._ParseAppYaml(basepath) rpc_server = self._GetRpcServer() @@ -1699,30 +2074,59 @@ class AppCfgApp(object): self.options.num_days, self.options.append, self.options.severity, - time.time()) + end_date, + self.options.vhost, + self.options.include_vhost) logs_requester.DownloadLogs() + def _ParseEndDate(self, date, time_func=time.time): + """Translates a user-readable end date to a POSIX timestamp. + + Args: + date: A utc date string as YYYY-MM-DD. + time_func: time.time() function for testing. + + Returns: + A POSIX timestamp representing the last moment of that day. + If no date is given, returns a timestamp representing now. + """ + if not date: + return time_func() + struct_time = time.strptime('%s' % date, '%Y-%m-%d') + return calendar.timegm(struct_time) + 86400 + def _RequestLogsOptions(self, parser): """Adds request_logs-specific options to 'parser'. Args: parser: An instance of OptionsParser. """ - parser.add_option("-n", "--num_days", type="int", dest="num_days", - action="store", default=None, - help="Number of days worth of log data to get. " - "The cut-off point is midnight UTC. " - "Use 0 to get all available logs. " - "Default is 1, unless --append is also given; " - "then the default is 0.") - parser.add_option("-a", "--append", dest="append", - action="store_true", default=False, - help="Append to existing file.") - parser.add_option("--severity", type="int", dest="severity", - action="store", default=None, - help="Severity of app-level log messages to get. " - "The range is 0 (DEBUG) through 4 (CRITICAL). " - "If omitted, only request logs are returned.") + parser.add_option('-n', '--num_days', type='int', dest='num_days', + action='store', default=None, + help='Number of days worth of log data to get. ' + 'The cut-off point is midnight UTC. ' + 'Use 0 to get all available logs. ' + 'Default is 1, unless --append is also given; ' + 'then the default is 0.') + parser.add_option('-a', '--append', dest='append', + action='store_true', default=False, + help='Append to existing file.') + parser.add_option('--severity', type='int', dest='severity', + action='store', default=None, + help='Severity of app-level log messages to get. ' + 'The range is 0 (DEBUG) through 4 (CRITICAL). ' + 'If omitted, only request logs are returned.') + parser.add_option('--vhost', type='string', dest='vhost', + action='store', default=None, + help='The virtual host of log messages to get. ' + 'If omitted, all log messages are returned.') + parser.add_option('--include_vhost', dest='include_vhost', + action='store_true', default=False, + help='Include virtual host in log messages.') + parser.add_option('--end_date', dest='end_date', + action='store', default='', + help='End date (as YYYY-MM-DD) of period for log data. ' + 'Defaults to today.') def CronInfo(self, now=None, output=sys.stdout): """Displays information about cron definitions. @@ -1732,25 +2136,25 @@ class AppCfgApp(object): output: Used for testing. """ if len(self.args) != 1: - self.parser.error("Expected a single <directory> argument.") + self.parser.error('Expected a single <directory> argument.') if now is None: now = datetime.datetime.now() basepath = self.args[0] cron_entries = self._ParseCronYaml(basepath) - if cron_entries: + if cron_entries and cron_entries.cron: for entry in cron_entries.cron: description = entry.description if not description: - description = "<no description>" - print >>output, "\n%s:\nURL: %s\nSchedule: %s" % (description, - entry.schedule, - entry.url) + description = '<no description>' + print >>output, '\n%s:\nURL: %s\nSchedule: %s' % (description, + entry.url, + entry.schedule) schedule = groctimespecification.GrocTimeSpecification(entry.schedule) matches = schedule.GetMatches(now, self.options.num_runs) for match in matches: - print >>output, "%s, %s from now" % ( - match.strftime("%Y-%m-%d %H:%M:%S"), match - now) + print >>output, '%s, %s from now' % ( + match.strftime('%Y-%m-%d %H:%M:%S'), match - now) def _CronInfoOptions(self, parser): """Adds cron_info-specific options to 'parser'. @@ -1758,10 +2162,232 @@ class AppCfgApp(object): Args: parser: An instance of OptionsParser. """ - parser.add_option("-n", "--num_runs", type="int", dest="num_runs", - action="store", default=5, - help="Number of runs of each cron job to display" - "Default is 5") + parser.add_option('-n', '--num_runs', type='int', dest='num_runs', + action='store', default=5, + help='Number of runs of each cron job to display' + 'Default is 5') + + def _CheckRequiredLoadOptions(self): + """Checks that upload/download options are present.""" + for option in ['filename', 'kind', 'config_file']: + if getattr(self.options, option) is None: + self.parser.error('Option \'%s\' is required.' % option) + if not self.options.url: + self.parser.error('You must have google.appengine.ext.remote_api.handler ' + 'assigned to an endpoint in app.yaml, or provide ' + 'the url of the handler via the \'url\' option.') + + def InferRemoteApiUrl(self, appyaml): + """Uses app.yaml to determine the remote_api endpoint. + + Args: + appyaml: A parsed app.yaml file. + + Returns: + The url of the remote_api endpoint as a string, or None + """ + handlers = appyaml.handlers + handler_suffix = 'remote_api/handler.py' + app_id = appyaml.application + for handler in handlers: + if hasattr(handler, 'script') and handler.script: + if handler.script.endswith(handler_suffix): + server = self.options.server + if server == 'appengine.google.com': + return 'http://%s.appspot.com%s' % (app_id, handler.url) + else: + return 'http://%s%s' % (server, handler.url) + return None + + def RunBulkloader(self, arg_dict): + """Invokes the bulkloader with the given keyword arguments. + + Args: + arg_dict: Dictionary of arguments to pass to bulkloader.Run(). + """ + try: + import sqlite3 + except ImportError: + logging.error('upload_data action requires SQLite3 and the python ' + 'sqlite3 module (included in python since 2.5).') + sys.exit(1) + + sys.exit(bulkloader.Run(arg_dict)) + + def _SetupLoad(self): + """Performs common verification and set up for upload and download.""" + if len(self.args) != 1: + self.parser.error('Expected <directory> argument.') + + basepath = self.args[0] + appyaml = self._ParseAppYaml(basepath) + + self.options.app_id = appyaml.application + + if not self.options.url: + url = self.InferRemoteApiUrl(appyaml) + if url is not None: + self.options.url = url + + self._CheckRequiredLoadOptions() + + if self.options.batch_size < 1: + self.parser.error('batch_size must be 1 or larger.') + + if verbosity == 1: + logging.getLogger().setLevel(logging.INFO) + self.options.debug = False + else: + logging.getLogger().setLevel(logging.DEBUG) + self.options.debug = True + + def _MakeLoaderArgs(self): + return dict([(arg_name, getattr(self.options, arg_name, None)) for + arg_name in ( + 'app_id', + 'url', + 'filename', + 'batch_size', + 'kind', + 'num_threads', + 'bandwidth_limit', + 'rps_limit', + 'http_limit', + 'db_filename', + 'config_file', + 'auth_domain', + 'has_header', + 'loader_opts', + 'log_file', + 'passin', + 'email', + 'debug', + 'exporter_opts', + 'mapper_opts', + 'result_db_filename', + 'mapper_opts', + 'dry_run', + 'dump', + 'restore', + )]) + + def PerformDownload(self, run_fn=None): + """Performs a datastore download via the bulkloader. + + Args: + run_fn: Function to invoke the bulkloader, used for testing. + """ + if run_fn is None: + run_fn = self.RunBulkloader + self._SetupLoad() + + StatusUpdate('Downloading data records.') + + args = self._MakeLoaderArgs() + args['download'] = True + args['has_header'] = False + args['map'] = False + args['dump'] = False + args['restore'] = False + + run_fn(args) + + def PerformUpload(self, run_fn=None): + """Performs a datastore upload via the bulkloader. + + Args: + run_fn: Function to invoke the bulkloader, used for testing. + """ + if run_fn is None: + run_fn = self.RunBulkloader + self._SetupLoad() + + StatusUpdate('Uploading data records.') + + args = self._MakeLoaderArgs() + args['download'] = False + args['map'] = False + args['dump'] = False + args['restore'] = False + + run_fn(args) + + def _PerformLoadOptions(self, parser): + """Adds options common to 'upload_data' and 'download_data'. + + Args: + parser: An instance of OptionsParser. + """ + parser.add_option('--filename', type='string', dest='filename', + action='store', + help='The name of the file containing the input data.' + ' (Required)') + parser.add_option('--config_file', type='string', dest='config_file', + action='store', + help='Name of the configuration file. (Required)') + parser.add_option('--kind', type='string', dest='kind', + action='store', + help='The kind of the entities to store. (Required)') + parser.add_option('--url', type='string', dest='url', + action='store', + help='The location of the remote_api endpoint.') + parser.add_option('--num_threads', type='int', dest='num_threads', + action='store', default=10, + help='Number of threads to upload records with.') + parser.add_option('--batch_size', type='int', dest='batch_size', + action='store', default=10, + help='Number of records to post in each request.') + parser.add_option('--bandwidth_limit', type='int', dest='bandwidth_limit', + action='store', default=250000, + help='The maximum bytes/second bandwidth for transfers.') + parser.add_option('--rps_limit', type='int', dest='rps_limit', + action='store', default=20, + help='The maximum records/second for transfers.') + parser.add_option('--http_limit', type='int', dest='http_limit', + action='store', default=8, + help='The maximum requests/second for transfers.') + parser.add_option('--db_filename', type='string', dest='db_filename', + action='store', + help='Name of the progress database file.') + parser.add_option('--auth_domain', type='string', dest='auth_domain', + action='store', default='gmail.com', + help='The name of the authorization domain to use.') + parser.add_option('--log_file', type='string', dest='log_file', + help='File to write bulkloader logs. If not supplied ' + 'then a new log file will be created, named: ' + 'bulkloader-log-TIMESTAMP.') + parser.add_option('--dry_run', action='store_true', + dest='dry_run', default=False, + help='Do not execute any remote_api calls') + + def _PerformUploadOptions(self, parser): + """Adds 'upload_data' specific options to the 'parser' passed in. + + Args: + parser: An instance of OptionsParser. + """ + self._PerformLoadOptions(parser) + parser.add_option('--has_header', dest='has_header', + action='store_true', default=False, + help='Whether the first line of the input file should be' + ' skipped') + parser.add_option('--loader_opts', type='string', dest='loader_opts', + help='A string to pass to the Loader.initialize method.') + + def _PerformDownloadOptions(self, parser): + """Adds 'download_data' specific options to the 'parser' passed in. + + Args: + parser: An instance of OptionsParser. + """ + self._PerformLoadOptions(parser) + parser.add_option('--exporter_opts', type='string', dest='exporter_opts', + help='A string to pass to the Exporter.initialize method.' + ) + parser.add_option('--result_db_filename', type='string', + dest='result_db_filename', + action='store', + help='Database to write entities to for download.') class Action(object): """Contains information about a command line action. @@ -1777,7 +2403,7 @@ class AppCfgApp(object): object. """ - def __init__(self, function, usage, short_desc, long_desc="", + def __init__(self, function, usage, short_desc, long_desc='', options=lambda obj, parser: None): """Initializer for the class attributes.""" self.function = function @@ -1790,22 +2416,28 @@ class AppCfgApp(object): """Invoke this Action on the specified AppCfg. This calls the function of the appropriate name on AppCfg, and - respects polymophic overrides.""" + respects polymophic overrides. + + Args: + appcfg: The appcfg to use. + Returns: + The result of the function call. + """ method = getattr(appcfg, self.function) return method() actions = { - "help": Action( - function="Help", - usage="%prog help <action>", - short_desc="Print help for a specific action."), + 'help': Action( + function='Help', + usage='%prog help <action>', + short_desc='Print help for a specific action.'), - "update": Action( - function="Update", - usage="%prog [options] update <directory>", + 'update': Action( + function='Update', + usage='%prog [options] update <directory>', options=_UpdateOptions, - short_desc="Create or update an app version.", + short_desc='Create or update an app version.', long_desc=""" Specify a directory that contains all of the files required by the app, and appcfg.py will create/update the app version referenced @@ -1813,28 +2445,35 @@ in the app.yaml file at the top level of that directory. appcfg.py will follow symlinks and recursively upload all files to the server. Temporary or source control files (e.g. foo~, .svn/*) will be skipped."""), + 'update_cron': Action( + function='UpdateCron', + usage='%prog [options] update_cron <directory>', + short_desc='Update application cron definitions.', + long_desc=""" +The 'update_cron' command will update any new, removed or changed cron +definitions from the optional cron.yaml file."""), - - - - - - - - - "update_indexes": Action( - function="UpdateIndexes", - usage="%prog [options] update_indexes <directory>", - short_desc="Update application indexes.", + 'update_indexes': Action( + function='UpdateIndexes', + usage='%prog [options] update_indexes <directory>', + short_desc='Update application indexes.', long_desc=""" The 'update_indexes' command will add additional indexes which are not currently in production as well as restart any indexes that were not completed."""), - "vacuum_indexes": Action( - function="VacuumIndexes", - usage="%prog [options] vacuum_indexes <directory>", + 'update_queues': Action( + function='UpdateQueues', + usage='%prog [options] update_queues <directory>', + short_desc='Update application task queue definitions.', + long_desc=""" +The 'update_queue' command will update any new, removed or changed task queue +definitions from the optional queue.yaml file."""), + + 'vacuum_indexes': Action( + function='VacuumIndexes', + usage='%prog [options] vacuum_indexes <directory>', options=_VacuumIndexesOptions, - short_desc="Delete unused indexes from application.", + short_desc='Delete unused indexes from application.', long_desc=""" The 'vacuum_indexes' command will help clean up indexes which are no longer in use. It does this by comparing the local index configuration with @@ -1842,34 +2481,51 @@ indexes that are actually defined on the server. If any indexes on the server do not exist in the index configuration file, the user is given the option to delete them."""), - "rollback": Action( - function="Rollback", - usage="%prog [options] rollback <directory>", - short_desc="Rollback an in-progress update.", + 'rollback': Action( + function='Rollback', + usage='%prog [options] rollback <directory>', + short_desc='Rollback an in-progress update.', long_desc=""" The 'update' command requires a server-side transaction. Use 'rollback' if you get an error message about another transaction being in progress and you are sure that there is no such transaction."""), - "request_logs": Action( - function="RequestLogs", - usage="%prog [options] request_logs <directory> <output_file>", + 'request_logs': Action( + function='RequestLogs', + usage='%prog [options] request_logs <directory> <output_file>', options=_RequestLogsOptions, - short_desc="Write request logs in Apache common log format.", + short_desc='Write request logs in Apache common log format.', long_desc=""" The 'request_logs' command exports the request logs from your application to a file. It will write Apache common log format records ordered chronologically. If output file is '-' stdout will be written."""), - - - - - - - - - + 'cron_info': Action( + function='CronInfo', + usage='%prog [options] cron_info <directory>', + options=_CronInfoOptions, + short_desc='Display information about cron jobs.', + long_desc=""" +The 'cron_info' command will display the next 'number' runs (default 5) for +each cron job defined in the cron.yaml file."""), + + 'upload_data': Action( + function='PerformUpload', + usage='%prog [options] upload_data <directory>', + options=_PerformUploadOptions, + short_desc='Upload data records to datastore.', + long_desc=""" +The 'upload_data' command translates input records into datastore entities and +uploads them into your application's datastore."""), + + 'download_data': Action( + function='PerformDownload', + usage='%prog [options] download_data <directory>', + options=_PerformDownloadOptions, + short_desc='Download entities from datastore.', + long_desc=""" +The 'download_data' command downloads datastore entities and writes them to +file as CSV or developer defined format."""), @@ -1877,16 +2533,16 @@ chronologically. If output file is '-' stdout will be written."""), def main(argv): - logging.basicConfig(format=("%(asctime)s %(levelname)s %(filename)s:" - "%(lineno)s %(message)s ")) + logging.basicConfig(format=('%(asctime)s %(levelname)s %(filename)s:' + '%(lineno)s %(message)s ')) try: result = AppCfgApp(argv).Run() if result: sys.exit(result) except KeyboardInterrupt: - StatusUpdate("Interrupted.") + StatusUpdate('Interrupted.') sys.exit(1) -if __name__ == "__main__": +if __name__ == '__main__': main(sys.argv) google_appengine/google/appengine/tools/appcfg.py @@ -41,6 +41,7 @@ try: except ImportError: pass +logger = logging.getLogger('google.appengine.tools.appengine_rpc') def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform): """Returns a 'User-agent' token for the host system platform. @@ -61,6 +62,33 @@ def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform): else: return "unknown" +def HttpRequestToString(req, include_data=True): + """Converts a urllib2.Request to a string. + + Args: + req: urllib2.Request + Returns: + Multi-line string representing the request. + """ + + headers = "" + for header in req.header_items(): + headers += "%s: %s\n" % (header[0], header[1]) + + template = ("%(method)s %(selector)s %(type)s/1.1\n" + "Host: %(host)s\n" + "%(headers)s") + if include_data: + template = template + "\n%(data)s" + + return template % { + 'method' : req.get_method(), + 'selector' : req.get_selector(), + 'type' : req.get_type().upper(), + 'host' : req.get_host(), + 'headers': headers, + 'data': req.get_data(), + } class ClientLoginError(urllib2.HTTPError): """Raised to indicate there was an error authenticating with ClientLogin.""" @@ -70,13 +98,16 @@ class ClientLoginError(urllib2.HTTPError): self.args = args self.reason = args["Error"] + def read(self): + return '%d %s: %s' % (self.code, self.msg, self.reason) + class AbstractRpcServer(object): """Provides a common interface for a simple RPC server.""" def __init__(self, host, auth_function, user_agent, source, host_override=None, extra_headers=None, save_cookies=False, - auth_tries=3, account_type=None): + auth_tries=3, account_type=None, debug_data=True, secure=True): """Creates a new HttpRpcServer. Args: @@ -95,13 +126,19 @@ class AbstractRpcServer(object): implement this functionality. Defaults to False. auth_tries: The number of times to attempt auth_function before failing. account_type: One of GOOGLE, HOSTED_OR_GOOGLE, or None for automatic. + debug_data: Whether debugging output should include data contents. """ + if secure: + self.scheme = "https" + else: + self.scheme = "http" self.host = host self.host_override = host_override self.auth_function = auth_function self.source = source self.authenticated = False self.auth_tries = auth_tries + self.debug_data = debug_data self.account_type = account_type @@ -115,9 +152,9 @@ class AbstractRpcServer(object): self.cookie_jar = cookielib.MozillaCookieJar() self.opener = self._GetOpener() if self.host_override: - logging.info("Server: %s; Host: %s", self.host, self.host_override) + logger.info("Server: %s; Host: %s", self.host, self.host_override) else: - logging.info("Server: %s", self.host) + logger.info("Server: %s", self.host) if ((self.host_override and self.host_override == "localhost") or self.host == "localhost" or self.host.startswith("localhost:")): @@ -200,8 +237,9 @@ class AbstractRpcServer(object): continue_location = "http://localhost/" args = {"continue": continue_location, "auth": auth_token} login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah") - req = self._CreateRequest("http://%s%s/login?%s" % - (self.host, login_path, urllib.urlencode(args))) + req = self._CreateRequest("%s://%s%s/login?%s" % + (self.scheme, self.host, login_path, + urllib.urlencode(args))) try: response = self.opener.open(req) except urllib2.HTTPError, e: @@ -291,30 +329,39 @@ class AbstractRpcServer(object): socket.setdefaulttimeout(timeout) try: tries = 0 + auth_tried = False while True: tries += 1 args = dict(kwargs) - url = "http://%s%s?%s" % (self.host, request_path, - urllib.urlencode(args)) + url = "%s://%s%s?%s" % (self.scheme, self.host, request_path, + urllib.urlencode(args)) req = self._CreateRequest(url=url, data=payload) req.add_header("Content-Type", content_type) req.add_header("X-appcfg-api-version", "1") try: + logger.debug('Sending HTTP request:\n%s' % + HttpRequestToString(req, include_data=self.debug_data)) f = self.opener.open(req) response = f.read() f.close() return response except urllib2.HTTPError, e: - logging.debug("Got http error, this is try #%s" % tries) + logger.debug("Got http error, this is try #%s" % tries) if tries > self.auth_tries: raise elif e.code == 401: + if auth_tried: + raise + auth_tried = True self._Authenticate() elif e.code >= 500 and e.code < 600: continue elif e.code == 302: + if auth_tried: + raise + auth_tried = True loc = e.info()["location"] - logging.debug("Got 302 redirect. Location: %s" % loc) + logger.debug("Got 302 redirect. Location: %s" % loc) if loc.startswith("https://www.google.com/accounts/ServiceLogin"): self._Authenticate() elif re.match(r"https://www.google.com/a/[a-z0-9.-]+/ServiceLogin", @@ -337,14 +384,14 @@ class HttpRpcServer(AbstractRpcServer): def _Authenticate(self): """Save the cookie jar after authentication.""" if cert_file_available and not uses_cert_verification: - logging.warn("ssl module not found. Without this the identity of the " - "remote host cannot be verified, and connections are NOT " - "secure. To fix this, please install the ssl module from " - "http://pypi.python.org/pypi/ssl") + logger.warn("ssl module not found. Without this the identity of the " + "remote host cannot be verified, and connections are NOT " + "secure. To fix this, please install the ssl module from " + "http://pypi.python.org/pypi/ssl") super(HttpRpcServer, self)._Authenticate() if self.cookie_jar.filename is not None and self.save_cookies: - logging.info("Saving authentication cookies to %s" % - self.cookie_jar.filename) + logger.info("Saving authentication cookies to %s" % + self.cookie_jar.filename) self.cookie_jar.save() def _GetOpener(self): @@ -369,19 +416,19 @@ class HttpRpcServer(AbstractRpcServer): try: self.cookie_jar.load() self.authenticated = True - logging.info("Loaded authentication cookies from %s" % - self.cookie_jar.filename) + logger.info("Loaded authentication cookies from %s" % + self.cookie_jar.filename) except (OSError, IOError, cookielib.LoadError), e: - logging.debug("Could not load authentication cookies; %s: %s", - e.__class__.__name__, e) + logger.debug("Could not load authentication cookies; %s: %s", + e.__class__.__name__, e) self.cookie_jar.filename = None else: try: fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600) os.close(fd) except (OSError, IOError), e: - logging.debug("Could not create authentication cookies file; %s: %s", - e.__class__.__name__, e) + logger.debug("Could not create authentication cookies file; %s: %s", + e.__class__.__name__, e) self.cookie_jar.filename = None opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) google_appengine/google/appengine/tools/appengine_rpc.py @@ -36,12 +36,20 @@ from google.appengine.tools import os_compat import __builtin__ import BaseHTTPServer import Cookie +import base64 import cStringIO import cgi import cgitb + +try: + import distutils.util +except ImportError: + pass + import dummy_thread import email.Utils import errno +import heapq import httplib import imp import inspect @@ -54,13 +62,13 @@ import os import pickle import pprint import random +import select import re import sre_compile import sre_constants import sre_parse -import mimetypes import socket import sys import time @@ -74,6 +82,7 @@ from google.pyglib import gexcept from google.appengine.api import apiproxy_stub_map from google.appengine.api import appinfo +from google.appengine.api import croninfo from google.appengine.api import datastore_admin from google.appengine.api import datastore_file_stub from google.appengine.api import mail_stub @@ -81,7 +90,11 @@ from google.appengine.api import urlfetch_stub from google.appengine.api import user_service_stub from google.appengine.api import yaml_errors from google.appengine.api.capabilities import capability_stub +from google.appengine.api.labs.taskqueue import taskqueue_stub from google.appengine.api.memcache import memcache_stub +from google.appengine.api.xmpp import xmpp_service_stub + +from google.appengine import dist from google.appengine.tools import dev_appserver_index from google.appengine.tools import dev_appserver_login @@ -100,11 +113,13 @@ MIDDLE_TEMPLATE = 'logging_console_middle.html' FOOTER_TEMPLATE = 'logging_console_footer.html' DEFAULT_ENV = { - 'GATEWAY_INTERFACE': 'CGI/1.1', - 'AUTH_DOMAIN': 'gmail.com', - 'TZ': 'UTC', + 'GATEWAY_INTERFACE': 'CGI/1.1', + 'AUTH_DOMAIN': 'gmail.com', + 'TZ': 'UTC', } +DEFAULT_SELECT_DELAY = 30.0 + for ext, mime_type in (('.asc', 'text/plain'), ('.diff', 'text/plain'), ('.csv', 'text/comma-separated-values'), @@ -113,24 +128,34 @@ for ext, mime_type in (('.asc', 'text/plain'), ('.wbmp', 'image/vnd.wap.wbmp')): mimetypes.add_type(mime_type, ext) -MAX_RUNTIME_RESPONSE_SIZE = 1 << 20 +MAX_RUNTIME_RESPONSE_SIZE = 10 << 20 MAX_REQUEST_SIZE = 10 * 1024 * 1024 +API_VERSION = '1' + +SITE_PACKAGES = os.path.normcase(os.path.join(os.path.dirname(os.__file__), + 'site-packages')) + + class Error(Exception): """Base-class for exceptions in this module.""" + class InvalidAppConfigError(Error): """The supplied application configuration file is invalid.""" + class AppConfigNotFoundError(Error): """Application configuration file not found.""" + class TemplatesNotLoadedError(Error): """Templates for the debugging console were not loaded.""" + def SplitURL(relative_url): """Splits a relative URL into its path and query-string components. @@ -143,7 +168,8 @@ def SplitURL(relative_url): script_name: Relative URL of the script that was accessed. query_string: String containing everything after the '?' character. """ - scheme, netloc, path, query, fragment = urlparse.urlsplit(relative_url) + (unused_scheme, unused_netloc, path, query, + unused_fragment) = urlparse.urlsplit(relative_url) return path, query @@ -166,6 +192,7 @@ def GetFullURL(server_name, server_port, relative_url): return 'http://%s%s' % (netloc, relative_url) + class URLDispatcher(object): """Base-class for handling HTTP requests.""" @@ -193,9 +220,32 @@ class URLDispatcher(object): outfile: File-like object where output data should be written. base_env_dict: Dictionary of CGI environment parameters if available. Defaults to None. + + Returns: + None if request handling is complete. + Tuple (path, headers, input_file) for an internal redirect: + path: Path of URL to redirect to. + headers: Headers to send to other dispatcher. + input_file: New input to send to new dispatcher. """ raise NotImplementedError + def EndRedirect(self, dispatched_output, original_output): + """Process the end of an internal redirect. + + This method is called after all subsequent dispatch requests have finished. + By default the output from the dispatched process is copied to the original. + + This will not be called on dispatchers that do not return an internal + redirect. + + Args: + dispatched_output: StringIO buffer containing the results from the + dispatched + original_output: The original output file. + """ + original_output.write(dispatched_output.read()) + class URLMatcher(object): """Matches an arbitrary URL using a list of URL patterns from an application. @@ -229,19 +279,23 @@ class URLMatcher(object): URL; False if anyone can access this URL. admin_only: True if the user must be a logged-in administrator to access the URL; False if anyone can access the URL. + + Raises: + TypeError: if dispatcher is not a URLDispatcher sub-class instance. + InvalidAppConfigError: if regex isn't valid. """ if not isinstance(dispatcher, URLDispatcher): - raise TypeError, 'dispatcher must be a URLDispatcher sub-class' + raise TypeError('dispatcher must be a URLDispatcher sub-class') if regex.startswith('^') or regex.endswith('$'): - raise InvalidAppConfigError, 'regex starts with "^" or ends with "$"' + raise InvalidAppConfigError('regex starts with "^" or ends with "$"') adjusted_regex = '^%s$' % regex try: url_re = re.compile(adjusted_regex) except re.error, e: - raise InvalidAppConfigError, 'regex invalid: %s' % e + raise InvalidAppConfigError('regex invalid: %s' % e) match_tuple = (url_re, dispatcher, path, requires_login, admin_only) self._url_patterns.append(match_tuple) @@ -256,6 +310,7 @@ class URLMatcher(object): Args: relative_url: Relative URL being accessed in a request. + split_url: Used for dependency injection. Returns: Tuple (dispatcher, matched_path, requires_login, admin_only), which are @@ -264,7 +319,7 @@ class URLMatcher(object): replaced using values matched by the URL pattern. If no match was found, dispatcher will be None. """ - adjusted_url, query_string = split_url(relative_url) + adjusted_url, unused_query_string = split_url(relative_url) for url_tuple in self._url_patterns: url_re, dispatcher, path, requires_login, admin_only = url_tuple @@ -287,6 +342,7 @@ class URLMatcher(object): return set([url_tuple[1] for url_tuple in self._url_patterns]) + class MatcherDispatcher(URLDispatcher): """Dispatcher across multiple URLMatcher instances.""" @@ -300,7 +356,8 @@ class MatcherDispatcher(URLDispatcher): Args: login_url: Relative URL which should be used for handling user logins. url_matchers: Sequence of URLMatcher objects. - get_user_info, login_redirect: Used for dependency injection. + get_user_info: Used for dependency injection. + login_redirect: Used for dependency injection. """ self._login_url = login_url self._url_matchers = tuple(url_matchers) @@ -321,37 +378,50 @@ class MatcherDispatcher(URLDispatcher): path variable supplied to this method is ignored. """ cookies = ', '.join(headers.getheaders('cookie')) - email, admin = self._get_user_info(cookies) + email_addr, admin, user_id = self._get_user_info(cookies) for matcher in self._url_matchers: - dispatcher, matched_path, requires_login, admin_only = matcher.Match(relative_url) + dispatcher, matched_path, requires_login, admin_only = matcher.Match( + relative_url) if dispatcher is None: continue logging.debug('Matched "%s" to %s with path %s', relative_url, dispatcher, matched_path) - if (requires_login or admin_only) and not email: + if (requires_login or admin_only) and not email_addr: logging.debug('Login required, redirecting user') - self._login_redirect( - self._login_url, - base_env_dict['SERVER_NAME'], - base_env_dict['SERVER_PORT'], - relative_url, - outfile) + self._login_redirect(self._login_url, + base_env_dict['SERVER_NAME'], + base_env_dict['SERVER_PORT'], + relative_url, + outfile) elif admin_only and not admin: outfile.write('Status: %d Not authorized\r\n' '\r\n' 'Current logged in user %s is not ' 'authorized to view this page.' - % (httplib.FORBIDDEN, email)) + % (httplib.FORBIDDEN, email_addr)) else: - dispatcher.Dispatch(relative_url, - matched_path, - headers, - infile, - outfile, - base_env_dict=base_env_dict) + forward = dispatcher.Dispatch(relative_url, + matched_path, + headers, + infile, + outfile, + base_env_dict=base_env_dict) + + if forward: + new_path, new_headers, new_input = forward + logging.info('Internal redirection to %s', new_path) + new_outfile = cStringIO.StringIO() + self.Dispatch(new_path, + None, + new_headers, + new_input, + new_outfile, + dict(base_env_dict)) + new_outfile.seek(0) + dispatcher.EndRedirect(new_outfile, outfile) return @@ -362,6 +432,7 @@ class MatcherDispatcher(URLDispatcher): % (httplib.NOT_FOUND, relative_url)) + class ApplicationLoggingHandler(logging.Handler): """Python Logging handler that displays the debugging console to users.""" @@ -436,7 +507,7 @@ class ApplicationLoggingHandler(logging.Handler): outfile: Output stream to which the console should be written if either a debug parameter was supplied or a logging cookie is present. """ - script_name, query_string = SplitURL(relative_url) + unused_script_name, query_string = SplitURL(relative_url) param_dict = cgi.parse_qs(query_string, True) cookie_dict = Cookie.SimpleCookie(env.get('HTTP_COOKIE', '')) if 'debug' not in param_dict and self._COOKIE_NAME not in cookie_dict: @@ -471,9 +542,11 @@ class ApplicationLoggingHandler(logging.Handler): _IGNORE_REQUEST_HEADERS = frozenset(['content-type', 'content-length', 'accept-encoding', 'transfer-encoding']) + def SetupEnvironment(cgi_path, relative_url, headers, + infile, split_url=SplitURL, get_user_info=dev_appserver_login.GetUserInfo): """Sets up environment variables for a CGI. @@ -482,6 +555,7 @@ def SetupEnvironment(cgi_path, cgi_path: Full file-system path to the CGI being executed. relative_url: Relative URL used to access the CGI. headers: Instance of mimetools.Message containing request headers. + infile: File-like object with input data from the request. split_url, get_user_info: Used for dependency injection. Returns: @@ -500,8 +574,9 @@ def SetupEnvironment(cgi_path, env['CONTENT_LENGTH'] = headers.getheader('content-length', '') cookies = ', '.join(headers.getheaders('cookie')) - email, admin = get_user_info(cookies) - env['USER_EMAIL'] = email + email_addr, admin, user_id = get_user_info(cookies) + env['USER_EMAIL'] = email_addr + env['USER_ID'] = user_id if admin: env['USER_IS_ADMIN'] = '1' @@ -511,24 +586,28 @@ def SetupEnvironment(cgi_path, adjusted_name = key.replace('-', '_').upper() env['HTTP_' + adjusted_name] = ', '.join(headers.getheaders(key)) - return env - + PAYLOAD_HEADER = 'HTTP_X_APPENGINE_DEVELOPMENT_PAYLOAD' + if PAYLOAD_HEADER in env: + del env[PAYLOAD_HEADER] + new_data = base64.standard_b64decode(infile.getvalue()) + infile.seek(0) + infile.truncate() + infile.write(new_data) + infile.seek(0) + env['CONTENT_LENGTH'] = str(len(new_data)) -def FakeTemporaryFile(*args, **kwargs): - """Fake for tempfile.TemporaryFile that just uses StringIO.""" - return cStringIO.StringIO() + return env def NotImplementedFake(*args, **kwargs): """Fake for methods/functions that are not implemented in the production environment. """ - raise NotImplementedError("This class/method is not available.") + raise NotImplementedError('This class/method is not available.') class NotImplementedFakeClass(object): - """Fake class for classes that are not implemented in the production - environment. + """Fake class for classes that are not implemented in the production env. """ __init__ = NotImplementedFake @@ -567,7 +646,7 @@ def ClearAllButEncodingsModules(module_dict): def FakeURandom(n): """Fake version of os.urandom.""" bytes = '' - for i in xrange(n): + for _ in range(n): bytes += chr(random.randint(0, 255)) return bytes @@ -577,13 +656,57 @@ def FakeUname(): return ('Linux', '', '', '', '') +def FakeUnlink(path): + """Fake version of os.unlink.""" + if os.path.isdir(path): + raise OSError(errno.ENOENT, "Is a directory", path) + else: + raise OSError(errno.EPERM, "Operation not permitted", path) + + +def FakeReadlink(path): + """Fake version of os.readlink.""" + raise OSError(errno.EINVAL, "Invalid argument", path) + + +def FakeAccess(path, mode): + """Fake version of os.access where only reads are supported.""" + if not os.path.exists(path) or mode != os.R_OK: + return False + else: + return True + + def FakeSetLocale(category, value=None, original_setlocale=locale.setlocale): """Fake version of locale.setlocale that only supports the default.""" if value not in (None, '', 'C', 'POSIX'): - raise locale.Error, 'locale emulation only supports "C" locale' + raise locale.Error('locale emulation only supports "C" locale') return original_setlocale(category, 'C') +def FakeOpen(filename, flags, mode=0777): + """Fake version of os.open.""" + raise OSError(errno.EPERM, "Operation not permitted", filename) + + +def FakeRename(src, dst): + """Fake version of os.rename.""" + raise OSError(errno.EPERM, "Operation not permitted", src) + + +def FakeUTime(path, times): + """Fake version of os.utime.""" + raise OSError(errno.EPERM, "Operation not permitted", path) + + +def FakeGetPlatform(): + """Fake distutils.util.get_platform on OS/X. Pass-through otherwise.""" + if sys.platform == 'darwin': + return 'macosx-' + else: + return distutils.util.get_platform() + + def IsPathInSubdirectories(filename, subdirectories, normcase=os.path.normcase): @@ -607,27 +730,27 @@ def IsPathInSubdirectories(filename, return False SHARED_MODULE_PREFIXES = set([ - 'google', - 'logging', - 'sys', - 'warnings', + 'google', + 'logging', + 'sys', + 'warnings', - 're', - 'sre_compile', - 'sre_constants', - 'sre_parse', + 're', + 'sre_compile', + 'sre_constants', + 'sre_parse', - 'wsgiref', + 'wsgiref', ]) NOT_SHARED_MODULE_PREFIXES = set([ - 'google.appengine.ext', + 'google.appengine.ext', ]) @@ -683,6 +806,21 @@ def SetupSharedModules(module_dict): return output_dict +def GeneratePythonPaths(*p): + """Generate all valid filenames for the given file. + + Args: + p: Positional args are the folders to the file and finally the file + without a suffix. + + Returns: + A list of strings representing the given path to a file with each valid + suffix for this python build. + """ + suffixes = imp.get_suffixes() + return [os.path.join(*p) + s for s, m, t in suffixes] + + class FakeFile(file): """File sub-class that enforces the security restrictions of the production environment. @@ -695,8 +833,8 @@ class FakeFile(file): if os.path.isfile(filename)) ALLOWED_DIRS = set([ - os.path.normcase(os.path.realpath(os.path.dirname(os.__file__))), - os.path.normcase(os.path.abspath(os.path.dirname(os.__file__))), + os.path.normcase(os.path.realpath(os.path.dirname(os.__file__))), + os.path.normcase(os.path.abspath(os.path.dirname(os.__file__))), ]) NOT_ALLOWED_DIRS = set([ @@ -704,41 +842,164 @@ class FakeFile(file): - os.path.normcase(os.path.join(os.path.dirname(os.__file__), - 'site-packages')) + SITE_PACKAGES, ]) ALLOWED_SITE_PACKAGE_DIRS = set( - os.path.normcase(os.path.abspath(os.path.join( - os.path.dirname(os.__file__), 'site-packages', path))) - for path in [ + os.path.normcase(os.path.abspath(os.path.join(SITE_PACKAGES, path))) + for path in [ + + ]) + + ALLOWED_SITE_PACKAGE_FILES = set( + os.path.normcase(os.path.abspath(os.path.join( + os.path.dirname(os.__file__), 'site-packages', path))) + for path in itertools.chain(*[ + + [os.path.join('Crypto')], + GeneratePythonPaths('Crypto', '__init__'), + [os.path.join('Crypto', 'Cipher')], + GeneratePythonPaths('Crypto', 'Cipher', '__init__'), + GeneratePythonPaths('Crypto', 'Cipher', 'AES'), + GeneratePythonPaths('Crypto', 'Cipher', 'ARC2'), + GeneratePythonPaths('Crypto', 'Cipher', 'ARC4'), + GeneratePythonPaths('Crypto', 'Cipher', 'Blowfish'), + GeneratePythonPaths('Crypto', 'Cipher', 'CAST'), + GeneratePythonPaths('Crypto', 'Cipher', 'DES'), + GeneratePythonPaths('Crypto', 'Cipher', 'DES3'), + GeneratePythonPaths('Crypto', 'Cipher', 'XOR'), + [os.path.join('Crypto', 'Hash')], + GeneratePythonPaths('Crypto', 'Hash', '__init__'), + GeneratePythonPaths('Crypto', 'Hash', 'HMAC'), + os.path.join('Crypto', 'Hash', 'MD2'), + os.path.join('Crypto', 'Hash', 'MD4'), + GeneratePythonPaths('Crypto', 'Hash', 'MD5'), + GeneratePythonPaths('Crypto', 'Hash', 'SHA'), + os.path.join('Crypto', 'Hash', 'SHA256'), + os.path.join('Crypto', 'Hash', 'RIPEMD'), + [os.path.join('Crypto', 'Protocol')], + GeneratePythonPaths('Crypto', 'Protocol', '__init__'), + GeneratePythonPaths('Crypto', 'Protocol', 'AllOrNothing'), + GeneratePythonPaths('Crypto', 'Protocol', 'Chaffing'), + [os.path.join('Crypto', 'PublicKey')], + GeneratePythonPaths('Crypto', 'PublicKey', '__init__'), + GeneratePythonPaths('Crypto', 'PublicKey', 'DSA'), + GeneratePythonPaths('Crypto', 'PublicKey', 'ElGamal'), + GeneratePythonPaths('Crypto', 'PublicKey', 'RSA'), + GeneratePythonPaths('Crypto', 'PublicKey', 'pubkey'), + GeneratePythonPaths('Crypto', 'PublicKey', 'qNEW'), + [os.path.join('Crypto', 'Util')], + GeneratePythonPaths('Crypto', 'Util', '__init__'), + GeneratePythonPaths('Crypto', 'Util', 'RFC1751'), + GeneratePythonPaths('Crypto', 'Util', 'number'), + GeneratePythonPaths('Crypto', 'Util', 'randpool'), + ])) - ]) + _original_file = file + _root_path = None _application_paths = None - _original_file = file + _skip_files = None + _static_file_config_matcher = None + + _allow_skipped_files = True + + _availability_cache = {} @staticmethod - def SetAllowedPaths(application_paths): - """Sets the root path of the application that is currently running. + def SetAllowedPaths(root_path, application_paths): + """Configures which paths are allowed to be accessed. Must be called at least once before any file objects are created in the hardened environment. Args: - root_path: Path to the root of the application. + root_path: Absolute path to the root of the application. + application_paths: List of additional paths that the application may + access, this must include the App Engine runtime but + not the Python library directories. """ FakeFile._application_paths = (set(os.path.realpath(path) for path in application_paths) | set(os.path.abspath(path) for path in application_paths)) + FakeFile._application_paths.add(root_path) + + FakeFile._root_path = os.path.join(root_path, '') + + FakeFile._availability_cache = {} + + @staticmethod + def SetAllowSkippedFiles(allow_skipped_files): + """Configures access to files matching FakeFile._skip_files. + + Args: + allow_skipped_files: Boolean whether to allow access to skipped files + """ + FakeFile._allow_skipped_files = allow_skipped_files + FakeFile._availability_cache = {} + + @staticmethod + def SetAllowedModule(name): + """Allow the use of a module based on where it is located. + + Meant to be used by use_library() so that it has a link back into the + trusted part of the interpreter. + + Args: + name: Name of the module to allow. + """ + stream, pathname, description = imp.find_module(name) + pathname = os.path.normcase(os.path.abspath(pathname)) + if stream: + stream.close() + FakeFile.ALLOWED_FILES.add(pathname) + FakeFile.ALLOWED_FILES.add(os.path.realpath(pathname)) + else: + assert description[2] == imp.PKG_DIRECTORY + if pathname.startswith(SITE_PACKAGES): + FakeFile.ALLOWED_SITE_PACKAGE_DIRS.add(pathname) + FakeFile.ALLOWED_SITE_PACKAGE_DIRS.add(os.path.realpath(pathname)) + else: + FakeFile.ALLOWED_DIRS.add(pathname) + FakeFile.ALLOWED_DIRS.add(os.path.realpath(pathname)) + + @staticmethod + def SetSkippedFiles(skip_files): + """Sets which files in the application directory are to be ignored. + + Must be called at least once before any file objects are created in the + hardened environment. + + Must be called whenever the configuration was updated. + + Args: + skip_files: Object with .match() method (e.g. compiled regexp). + """ + FakeFile._skip_files = skip_files + FakeFile._availability_cache = {} + + @staticmethod + def SetStaticFileConfigMatcher(static_file_config_matcher): + """Sets StaticFileConfigMatcher instance for checking if a file is static. + + Must be called at least once before any file objects are created in the + hardened environment. + + Must be called whenever the configuration was updated. + + Args: + static_file_config_matcher: StaticFileConfigMatcher instance. + """ + FakeFile._static_file_config_matcher = static_file_config_matcher + FakeFile._availability_cache = {} @staticmethod def IsFileAccessible(filename, normcase=os.path.normcase): """Determines if a file's path is accessible. - SetAllowedPaths() must be called before this method or else all file - accesses will raise an error. + SetAllowedPaths(), SetSkippedFiles() and SetStaticFileConfigMatcher() must + be called before this method or else all file accesses will raise an error. Args: filename: Path of the file to check (relative or absolute). May be a @@ -751,22 +1012,61 @@ class FakeFile(file): """ logical_filename = normcase(os.path.abspath(filename)) + result = FakeFile._availability_cache.get(logical_filename) + if result is None: + result = FakeFile._IsFileAccessibleNoCache(logical_filename, + normcase=normcase) + FakeFile._availability_cache[logical_filename] = result + return result + + @staticmethod + def _IsFileAccessibleNoCache(logical_filename, normcase=os.path.normcase): + """Determines if a file's path is accessible. + + This is an internal part of the IsFileAccessible implementation. + + Args: + logical_filename: Absolute path of the file to check. + normcase: Used for dependency injection. + + Returns: + True if the file is accessible, False otherwise. + """ + logical_dirfakefile = logical_filename if os.path.isdir(logical_filename): - logical_filename = os.path.join(logical_filename, 'foo') + logical_dirfakefile = os.path.join(logical_filename, 'foo') + + if IsPathInSubdirectories(logical_dirfakefile, [FakeFile._root_path], + normcase=normcase): + relative_filename = logical_dirfakefile[len(FakeFile._root_path):] + + if (not FakeFile._allow_skipped_files and + FakeFile._skip_files.match(relative_filename)): + logging.warning('Blocking access to skipped file "%s"', + logical_filename) + return False + + if FakeFile._static_file_config_matcher.IsStaticFile(relative_filename): + logging.warning('Blocking access to static file "%s"', + logical_filename) + return False if logical_filename in FakeFile.ALLOWED_FILES: return True - if IsPathInSubdirectories(logical_filename, + if logical_filename in FakeFile.ALLOWED_SITE_PACKAGE_FILES: + return True + + if IsPathInSubdirectories(logical_dirfakefile, FakeFile.ALLOWED_SITE_PACKAGE_DIRS, normcase=normcase): return True allowed_dirs = FakeFile._application_paths | FakeFile.ALLOWED_DIRS - if (IsPathInSubdirectories(logical_filename, + if (IsPathInSubdirectories(logical_dirfakefile, allowed_dirs, normcase=normcase) and - not IsPathInSubdirectories(logical_filename, + not IsPathInSubdirectories(logical_dirfakefile, FakeFile.NOT_ALLOWED_DIRS, normcase=normcase)): return True @@ -779,11 +1079,15 @@ class FakeFile(file): raise IOError('invalid mode: %s' % mode) if not FakeFile.IsFileAccessible(filename): - raise IOError(errno.EACCES, 'file not accessible') + raise IOError(errno.EACCES, 'file not accessible', filename) super(FakeFile, self).__init__(filename, mode, bufsize, **kwargs) +from google.appengine.dist import _library +_library.SetAllowedModule = FakeFile.SetAllowedModule + + class RestrictedPathFunction(object): """Enforces access restrictions for functions that have a file or directory path as their first argument.""" @@ -803,7 +1107,7 @@ class RestrictedPathFunction(object): """Enforces access permissions for the function passed to the constructor. """ if not FakeFile.IsFileAccessible(path): - raise OSError(errno.EACCES, 'path not accessible') + raise OSError(errno.EACCES, 'path not accessible', path) return self._original_func(path, *args, **kwargs) @@ -821,6 +1125,7 @@ def GetSubmoduleName(fullname): return fullname.rsplit('.', 1)[-1] + class CouldNotFindModuleError(ImportError): """Raised when a module could not be found. @@ -830,10 +1135,19 @@ class CouldNotFindModuleError(ImportError): def Trace(func): - """Decorator that logs the call stack of the HardenedModulesHook class as + """Call stack logging decorator for HardenedModulesHook class. + + This decorator logs the call stack of the HardenedModulesHook class as it executes, indenting logging messages based on the current stack depth. + + Args: + func: the function to decorate. + + Returns: + The decorated function. """ - def decorate(self, *args, **kwargs): + + def Decorate(self, *args, **kwargs): args_to_show = [] if args is not None: args_to_show.extend(str(argument) for argument in args) @@ -851,7 +1165,7 @@ def Trace(func): self._indent_level -= 1 self.log('Exiting %s(%s)', func.func_name, args_string) - return decorate + return Decorate class HardenedModulesHook(object): @@ -887,198 +1201,231 @@ class HardenedModulesHook(object): indent = self._indent_level * ' ' print >>sys.stderr, indent + (message % args) - EMPTY_MODULE_FILE = '<empty module>' - _WHITE_LIST_C_MODULES = [ - 'array', - 'binascii', - 'bz2', - 'cmath', - 'collections', - 'crypt', - 'cStringIO', - 'datetime', - 'errno', - 'exceptions', - 'gc', - 'itertools', - 'math', - 'md5', - 'operator', - 'posix', - 'posixpath', - 'pyexpat', - 'sha', - 'struct', - 'sys', - 'time', - 'timing', - 'unicodedata', - 'zlib', - '_bisect', - '_codecs', - '_codecs_cn', - '_codecs_hk', - '_codecs_iso2022', - '_codecs_jp', - '_codecs_kr', - '_codecs_tw', - '_collections', - '_csv', - '_elementtree', - '_functools', - '_hashlib', - '_heapq', - '_locale', - '_lsprof', - '_md5', - '_multibytecodec', - '_random', - '_sha', - '_sha256', - '_sha512', - '_sre', - '_struct', - '_types', - '_weakref', - '__main__', + 'py_streamhtmlparser', + 'AES', + 'ARC2', + 'ARC4', + 'Blowfish', + 'CAST', + 'DES', + 'DES3', + 'MD2', + 'MD4', + 'RIPEMD', + 'SHA256', + 'XOR', + + '_Crypto_Cipher__AES', + '_Crypto_Cipher__ARC2', + '_Crypto_Cipher__ARC4', + '_Crypto_Cipher__Blowfish', + '_Crypto_Cipher__CAST', + '_Crypto_Cipher__DES', + '_Crypto_Cipher__DES3', + '_Crypto_Cipher__XOR', + '_Crypto_Hash__MD2', + '_Crypto_Hash__MD4', + '_Crypto_Hash__RIPEMD', + '_Crypto_Hash__SHA256', + 'array', + 'binascii', + 'bz2', + 'cmath', + 'collections', + 'crypt', + 'cStringIO', + 'datetime', + 'errno', + 'exceptions', + 'gc', + 'itertools', + 'math', + 'md5', + 'operator', + 'posix', + 'posixpath', + 'pyexpat', + 'sha', + 'struct', + 'sys', + 'time', + 'timing', + 'unicodedata', + 'zlib', + '_ast', + '_bisect', + '_codecs', + '_codecs_cn', + '_codecs_hk', + '_codecs_iso2022', + '_codecs_jp', + '_codecs_kr', + '_codecs_tw', + '_collections', + '_csv', + '_elementtree', + '_functools', + '_hashlib', + '_heapq', + '_locale', + '_lsprof', + '_md5', + '_multibytecodec', + '_random', + '_sha', + '_sha256', + '_sha512', + '_sre', + '_struct', + '_types', + '_weakref', + '__main__', ] + __CRYPTO_CIPHER_ALLOWED_MODULES = [ + 'MODE_CBC', + 'MODE_CFB', + 'MODE_CTR', + 'MODE_ECB', + 'MODE_OFB', + 'block_size', + 'key_size', + 'new', + ] _WHITE_LIST_PARTIAL_MODULES = { - 'gc': [ - 'enable', - 'disable', - 'isenabled', - 'collect', - 'get_debug', - 'set_threshold', - 'get_threshold', - 'get_count' - ], - - - - 'os': [ - 'altsep', - 'curdir', - 'defpath', - 'devnull', - 'environ', - 'error', - 'extsep', - 'EX_NOHOST', - 'EX_NOINPUT', - 'EX_NOPERM', - 'EX_NOUSER', - 'EX_OK', - 'EX_OSERR', - 'EX_OSFILE', - 'EX_PROTOCOL', - 'EX_SOFTWARE', - 'EX_TEMPFAIL', - 'EX_UNAVAILABLE', - 'EX_USAGE', - 'F_OK', - 'getcwd', - 'getcwdu', - 'getenv', - 'listdir', - 'lstat', - 'name', - 'NGROUPS_MAX', - 'O_APPEND', - 'O_CREAT', - 'O_DIRECT', - 'O_DIRECTORY', - 'O_DSYNC', - 'O_EXCL', - 'O_LARGEFILE', - 'O_NDELAY', - 'O_NOCTTY', - 'O_NOFOLLOW', - 'O_NONBLOCK', - 'O_RDONLY', - 'O_RDWR', - 'O_RSYNC', - 'O_SYNC', - 'O_TRUNC', - 'O_WRONLY', - 'pardir', - 'path', - 'pathsep', - 'R_OK', - 'SEEK_CUR', - 'SEEK_END', - 'SEEK_SET', - 'sep', - 'stat', - 'stat_float_times', - 'stat_result', - 'strerror', - 'TMP_MAX', - 'urandom', - 'walk', - 'WCOREDUMP', - 'WEXITSTATUS', - 'WIFEXITED', - 'WIFSIGNALED', - 'WIFSTOPPED', - 'WNOHANG', - 'WSTOPSIG', - 'WTERMSIG', - 'WUNTRACED', - 'W_OK', - 'X_OK', - ], + 'Crypto.Cipher.AES': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.ARC2': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.Blowfish': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.CAST': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.DES': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.DES3': __CRYPTO_CIPHER_ALLOWED_MODULES, + + 'gc': [ + 'enable', + 'disable', + 'isenabled', + 'collect', + 'get_debug', + 'set_threshold', + 'get_threshold', + 'get_count' + ], + + + + 'os': [ + 'access', + 'altsep', + 'curdir', + 'defpath', + 'devnull', + 'environ', + 'error', + 'extsep', + 'EX_NOHOST', + 'EX_NOINPUT', + 'EX_NOPERM', + 'EX_NOUSER', + 'EX_OK', + 'EX_OSERR', + 'EX_OSFILE', + 'EX_PROTOCOL', + 'EX_SOFTWARE', + 'EX_TEMPFAIL', + 'EX_UNAVAILABLE', + 'EX_USAGE', + 'F_OK', + 'getcwd', + 'getcwdu', + 'getenv', + 'listdir', + 'lstat', + 'name', + 'NGROUPS_MAX', + 'O_APPEND', + 'O_CREAT', + 'O_DIRECT', + 'O_DIRECTORY', + 'O_DSYNC', + 'O_EXCL', + 'O_LARGEFILE', + 'O_NDELAY', + 'O_NOCTTY', + 'O_NOFOLLOW', + 'O_NONBLOCK', + 'O_RDONLY', + 'O_RDWR', + 'O_RSYNC', + 'O_SYNC', + 'O_TRUNC', + 'O_WRONLY', + 'open', + 'pardir', + 'path', + 'pathsep', + 'R_OK', + 'readlink', + 'remove', + 'rename', + 'SEEK_CUR', + 'SEEK_END', + 'SEEK_SET', + 'sep', + 'stat', + 'stat_float_times', + 'stat_result', + 'strerror', + 'TMP_MAX', + 'unlink', + 'urandom', + 'utime', + 'walk', + 'WCOREDUMP', + 'WEXITSTATUS', + 'WIFEXITED', + 'WIFSIGNALED', + 'WIFSTOPPED', + 'WNOHANG', + 'WSTOPSIG', + 'WTERMSIG', + 'WUNTRACED', + 'W_OK', + 'X_OK', + ], } - _EMPTY_MODULES = [ - 'imp', - 'ftplib', - 'select', - 'socket', - 'tempfile', - ] - _MODULE_OVERRIDES = { - 'locale': { - 'setlocale': FakeSetLocale, - }, - - 'os': { - 'listdir': RestrictedPathFunction(os.listdir), - - 'lstat': RestrictedPathFunction(os.stat), - 'stat': RestrictedPathFunction(os.stat), - 'uname': FakeUname, - 'urandom': FakeURandom, - }, - - 'socket': { - 'AF_INET': None, - 'SOCK_STREAM': None, - 'SOCK_DGRAM': None, - '_GLOBAL_DEFAULT_TIMEOUT': getattr(socket, '_GLOBAL_DEFAULT_TIMEOUT', - None), - }, - - 'tempfile': { - 'TemporaryFile': FakeTemporaryFile, - 'gettempdir': NotImplementedFake, - 'gettempprefix': NotImplementedFake, - 'mkdtemp': NotImplementedFake, - 'mkstemp': NotImplementedFake, - 'mktemp': NotImplementedFake, - 'NamedTemporaryFile': NotImplementedFake, - 'tempdir': NotImplementedFake, - }, + 'locale': { + 'setlocale': FakeSetLocale, + }, + + 'os': { + 'access': FakeAccess, + 'listdir': RestrictedPathFunction(os.listdir), + + 'lstat': RestrictedPathFunction(os.stat), + 'open': FakeOpen, + 'readlink': FakeReadlink, + 'remove': FakeUnlink, + 'rename': FakeRename, + 'stat': RestrictedPathFunction(os.stat), + 'uname': FakeUname, + 'unlink': FakeUnlink, + 'urandom': FakeURandom, + 'utime': FakeUTime, + }, + + 'distutils.util': { + 'get_platform': FakeGetPlatform, + }, } _ENABLED_FILE_TYPES = ( - imp.PKG_DIRECTORY, - imp.PY_SOURCE, - imp.PY_COMPILED, - imp.C_BUILTIN, + imp.PKG_DIRECTORY, + imp.PY_SOURCE, + imp.PY_COMPILED, + imp.C_BUILTIN, ) def __init__(self, @@ -1107,8 +1454,7 @@ class HardenedModulesHook(object): @Trace def find_module(self, fullname, path=None): """See PEP 302.""" - if (fullname in ('cPickle', 'thread') or - fullname in HardenedModulesHook._EMPTY_MODULES): + if fullname in ('cPickle', 'thread'): return self search_path = path @@ -1116,7 +1462,8 @@ class HardenedModulesHook(object): try: for index, current_module in enumerate(all_modules): current_module_fullname = '.'.join(all_modules[:index + 1]) - if current_module_fullname == fullname: + if (current_module_fullname == fullname and not + self.StubModuleExists(fullname)): self.FindModuleRestricted(current_module, current_module_fullname, search_path) @@ -1135,6 +1482,21 @@ class HardenedModulesHook(object): return self + def StubModuleExists(self, name): + """Check if the named module has a stub replacement.""" + if name in sys.builtin_module_names: + name = 'py_%s' % name + if name in dist.__all__: + return True + return False + + def ImportStubModule(self, name): + """Import the stub module replacement for the specified module.""" + if name in sys.builtin_module_names: + name = 'py_%s' % name + module = __import__(dist.__name__, {}, {}, [name]) + return getattr(module, name) + @Trace def FixModule(self, module): """Prunes and overrides restricted module attributes. @@ -1334,9 +1696,7 @@ class HardenedModulesHook(object): """ module = self._imp.new_module(submodule_fullname) - if submodule_fullname in self._EMPTY_MODULES: - module.__file__ = self.EMPTY_MODULE_FILE - elif submodule_fullname == 'thread': + if submodule_fullname == 'thread': module.__dict__.update(self._dummy_thread.__dict__) module.__name__ = 'thread' elif submodule_fullname == 'cPickle': @@ -1344,7 +1704,8 @@ class HardenedModulesHook(object): module.__name__ = 'cPickle' elif submodule_fullname == 'os': module.__dict__.update(self._os.__dict__) - self._module_dict['os.path'] = module.path + elif self.StubModuleExists(submodule_fullname): + module = self.ImportStubModule(submodule_fullname) else: source_file, pathname, description = self.FindModuleRestricted(submodule, submodule_fullname, search_path) module = self.LoadModuleRestricted(submodule_fullname, @@ -1357,6 +1718,12 @@ class HardenedModulesHook(object): if submodule_fullname not in self._module_dict: self._module_dict[submodule_fullname] = module + if submodule_fullname == 'os': + os_path_name = module.path.__name__ + os_path = self.FindAndLoadModule(os_path_name, os_path_name, search_path) + self._module_dict['os.path'] = os_path + module.__dict__['path'] = os_path + return module @Trace @@ -1485,6 +1852,7 @@ class HardenedModulesHook(object): return compile(source_code, full_path, 'exec') + def ModuleHasValidMainFunction(module): """Determines if a module has a main function that takes no arguments. @@ -1498,7 +1866,8 @@ def ModuleHasValidMainFunction(module): True if the module has a valid, reusable main function; False otherwise. """ if hasattr(module, 'main') and type(module.main) is types.FunctionType: - arg_names, var_args, var_kwargs, default_values = inspect.getargspec(module.main) + arg_names, var_args, var_kwargs, default_values = inspect.getargspec( + module.main) if len(arg_names) == 0: return True if default_values is not None and len(arg_names) == len(default_values): @@ -1541,6 +1910,7 @@ def FindMissingInitFiles(cgi_path, module_fullname, isfile=os.path.isfile): cgi_path: Absolute path of the CGI module file on disk. module_fullname: Fully qualified Python module name used to import the cgi_path module. + isfile: Used for testing. Returns: List containing the paths to the missing __init__.py files. @@ -1598,7 +1968,7 @@ def LoadTargetModule(handler_path, module_fullname = GetScriptModuleName(handler_path) script_module = module_dict.get(module_fullname) module_code = None - if script_module != None and ModuleHasValidMainFunction(script_module): + if script_module is not None and ModuleHasValidMainFunction(script_module): logging.debug('Reusing main() function of module "%s"', module_fullname) else: if script_module is None: @@ -1607,7 +1977,8 @@ def LoadTargetModule(handler_path, try: module_code = import_hook.get_code(module_fullname) - full_path, search_path, submodule = import_hook.GetModuleInfo(module_fullname) + full_path, search_path, submodule = ( + import_hook.GetModuleInfo(module_fullname)) script_module.__file__ = full_path if search_path is not None: script_module.__path__ = search_path @@ -1618,7 +1989,7 @@ def LoadTargetModule(handler_path, import_error_message += ': ' + str(exc_value) logging.exception('Encountered error loading module "%s": %s', - module_fullname, import_error_message) + module_fullname, import_error_message) missing_inits = FindMissingInitFiles(cgi_path, module_fullname) if missing_inits: logging.warning('Missing package initialization files: %s', @@ -1652,8 +2023,10 @@ def LoadTargetModule(handler_path, def ExecuteOrImportScript(handler_path, cgi_path, import_hook): - """Executes a CGI script by importing it as a new module; possibly reuses - the module's main() function if it is defined and takes no arguments. + """Executes a CGI script by importing it as a new module. + + This possibly reuses the module's main() function if it is defined and + takes no arguments. Basic technique lifted from PEP 338 and Python2.5's runpy module. See: http://www.python.org/dev/peps/pep-0338/ @@ -1761,7 +2134,7 @@ def ExecuteCGI(root_path, ClearAllButEncodingsModules(sys.modules) sys.modules.update(module_dict) sys.argv = [cgi_path] - sys.stdin = infile + sys.stdin = cStringIO.StringIO(infile.getvalue()) sys.stdout = outfile os.environ.clear() os.environ.update(env) @@ -1861,7 +2234,7 @@ class CGIDispatcher(URLDispatcher): if base_env_dict: env.update(base_env_dict) cgi_path = self._path_adjuster.AdjustPath(path) - env.update(self._setup_env(cgi_path, relative_url, headers)) + env.update(self._setup_env(cgi_path, relative_url, headers, infile)) self._exec_cgi(self._root_path, path, cgi_path, @@ -1923,6 +2296,7 @@ class LocalCGIDispatcher(CGIDispatcher): return 'Local CGI dispatcher for %s' % self._cgi_func + class PathAdjuster(object): """Adjusts application file paths to paths relative to the application or external library directories.""" @@ -1936,8 +2310,10 @@ class PathAdjuster(object): self._root_path = os.path.abspath(root_path) def AdjustPath(self, path): - """Adjusts application file path to paths relative to the application or - external library directories. + """Adjusts application file paths to relative to the application. + + More precisely this method adjusts application file path to paths + relative to the application or external library directories. Handler paths that start with $PYTHON_LIB will be converted to paths relative to the google directory. @@ -1957,6 +2333,7 @@ class PathAdjuster(object): return path + class StaticFileConfigMatcher(object): """Keeps track of file/directory specific application configuration. @@ -2004,7 +2381,7 @@ class StaticFileConfigMatcher(object): path = entry.static_dir if path[-1] == '/': path = path[:-1] - regex = re.escape(path) + r'/(.*)' + regex = re.escape(path + os.path.sep) + r'(.*)' try: path_re = re.compile(regex) @@ -2021,6 +2398,20 @@ class StaticFileConfigMatcher(object): self._patterns.append((path_re, entry.mime_type, expiration)) + def IsStaticFile(self, path): + """Tests if the given path points to a "static" file. + + Args: + path: String containing the file's path relative to the app. + + Returns: + Boolean, True if the file was configured to be static. + """ + for (path_re, _, _) in self._patterns: + if path_re.match(path): + return True + return False + def GetMimeType(self, path): """Returns the mime type that we should use when serving the specified file. @@ -2031,13 +2422,13 @@ class StaticFileConfigMatcher(object): String containing the mime type to use. Will be 'application/octet-stream' if we have no idea what it should be. """ - for (path_re, mime_type, expiration) in self._patterns: - if mime_type is not None: + for (path_re, mimetype, unused_expiration) in self._patterns: + if mimetype is not None: the_match = path_re.match(path) if the_match: - return mime_type + return mimetype - filename, extension = os.path.splitext(path) + unused_filename, extension = os.path.splitext(path) return mimetypes.types_map.get(extension, 'application/octet-stream') def GetExpiration(self, path): @@ -2049,7 +2440,7 @@ class StaticFileConfigMatcher(object): Returns: Integer number of seconds to be used for browser cache expiration time. """ - for (path_re, mime_type, expiration) in self._patterns: + for (path_re, unused_mimetype, expiration) in self._patterns: the_match = path_re.match(path) if the_match: return expiration @@ -2058,6 +2449,7 @@ class StaticFileConfigMatcher(object): + def ReadDataFile(data_path, openfile=file): """Reads a file on disk, returning a corresponding HTTP status and data. @@ -2143,8 +2535,25 @@ _IGNORE_RESPONSE_HEADERS = frozenset([ ]) -def RewriteResponse(response_file): - """Interprets server-side headers and adjusts the HTTP response accordingly. +def IgnoreHeadersRewriter(status_code, status_message, headers, body): + """Ignore specific response headers. + + Certain response headers cannot be modified by an Application. For a + complete list of these headers please see: + + http://code.google.com/appengine/docs/webapp/responseclass.html#Disallowed_HTTP_Response_Headers + + This rewriter simply removes those headers. + """ + for h in _IGNORE_RESPONSE_HEADERS: + if h in headers: + del headers[h] + + return status_code, status_message, headers, body + + +def ParseStatusRewriter(status_code, status_message, headers, body): + """Parse status header, if it exists. Handles the server-side 'status' header, which instructs the server to change the HTTP response code accordingly. Handles the 'location' header, which @@ -2154,30 +2563,7 @@ def RewriteResponse(response_file): If the 'status' header supplied by the client is invalid, this method will set the response to a 500 with an error message as content. - - Args: - response_file: File-like object containing the full HTTP response including - the response code, all headers, and the request body. - gmtime: Function which returns current time in a format matching standard - time.gmtime(). - - Returns: - Tuple (status_code, status_message, header, body) where: - status_code: Integer HTTP response status (e.g., 200, 302, 404, 500) - status_message: String containing an informational message about the - response code, possibly derived from the 'status' header, if supplied. - header: String containing the HTTP headers of the response, without - a trailing new-line (CRLF). - body: String containing the body of the response. """ - headers = mimetools.Message(response_file) - - for h in _IGNORE_RESPONSE_HEADERS: - if h in headers: - del headers[h] - - response_status = '%d Good to go' % httplib.OK - location_value = headers.getheader('location') status_value = headers.getheader('status') if status_value: @@ -2185,9 +2571,8 @@ def RewriteResponse(response_file): del headers['status'] elif location_value: response_status = '%d Redirecting' % httplib.FOUND - - if not 'Cache-Control' in headers: - headers['Cache-Control'] = 'no-cache' + else: + return status_code, status_message, headers, body status_parts = response_status.split(' ', 1) status_code, status_message = (status_parts + [''])[:2] @@ -2195,11 +2580,121 @@ def RewriteResponse(response_file): status_code = int(status_code) except ValueError: status_code = 500 - body = 'Error: Invalid "status" header value returned.' - else: - body = response_file.read() + body = cStringIO.StringIO('Error: Invalid "status" header value returned.') + + return status_code, status_message, headers, body - headers['Content-Length'] = str(len(body)) + +def CacheRewriter(status_code, status_message, headers, body): + """Update the cache header.""" + if not 'Cache-Control' in headers: + headers['Cache-Control'] = 'no-cache' + if not 'Expires' in headers: + headers['Expires'] = 'Fri, 01 Jan 1990 00:00:00 GMT' + return status_code, status_message, headers, body + + +def ContentLengthRewriter(status_code, status_message, headers, body): + """Rewrite the Content-Length header. + + Even though Content-Length is not a user modifiable header, App Engine + sends a correct Content-Length to the user based on the actual response. + """ + current_position = body.tell() + body.seek(0, 2) + + headers['Content-Length'] = str(body.tell() - current_position) + body.seek(current_position) + return status_code, status_message, headers, body + + +def CreateResponseRewritersChain(): + """Create the default response rewriter chain. + + A response rewriter is the a function that gets a final chance to change part + of the dev_appservers response. A rewriter is not like a dispatcher in that + it is called after every request has been handled by the dispatchers + regardless of which dispatcher was used. + + The order in which rewriters are registered will be the order in which they + are used to rewrite the response. Modifications from earlier rewriters + are used as input to later rewriters. + + A response rewriter is a function that can rewrite the request in any way. + Thefunction can returned modified values or the original values it was + passed. + + A rewriter function has the following parameters and return values: + + Args: + status_code: Status code of response from dev_appserver or previous + rewriter. + status_message: Text corresponding to status code. + headers: mimetools.Message instance with parsed headers. NOTE: These + headers can contain its own 'status' field, but the default + dev_appserver implementation will remove this. Future rewriters + should avoid re-introducing the status field and return new codes + instead. + body: File object containing the body of the response. This position of + this file may not be at the start of the file. Any content before the + files position is considered not to be part of the final body. + + Returns: + status_code: Rewritten status code or original. + status_message: Rewritter message or original. + headers: Rewritten/modified headers or original. + body: Rewritten/modified body or original. + + Returns: + List of response rewriters. + """ + return [IgnoreHeadersRewriter, + ParseStatusRewriter, + CacheRewriter, + ContentLengthRewriter, + ] + + +def RewriteResponse(response_file, response_rewriters=None): + """Allows final rewrite of dev_appserver response. + + This function receives the unparsed HTTP response from the application + or internal handler, parses out the basic structure and feeds that structure + in to a chain of response rewriters. + + It also makes sure the final HTTP headers are properly terminated. + + For more about response rewriters, please see documentation for + CreateResponeRewritersChain. + + Args: + response_file: File-like object containing the full HTTP response including + the response code, all headers, and the request body. + response_rewriters: A list of response rewriters. If none is provided it + will create a new chain using CreateResponseRewritersChain. + + Returns: + Tuple (status_code, status_message, header, body) where: + status_code: Integer HTTP response status (e.g., 200, 302, 404, 500) + status_message: String containing an informational message about the + response code, possibly derived from the 'status' header, if supplied. + header: String containing the HTTP headers of the response, without + a trailing new-line (CRLF). + body: String containing the body of the response. + """ + if response_rewriters is None: + response_rewriters = CreateResponseRewritersChain() + + status_code = 200 + status_message = 'Good to go' + headers = mimetools.Message(response_file) + + for response_rewriter in response_rewriters: + status_code, status_message, headers, response_file = response_rewriter( + status_code, + status_message, + headers, + response_file) header_list = [] for header in headers.headers: @@ -2208,7 +2703,8 @@ def RewriteResponse(response_file): header_list.append(header) header_data = '\r\n'.join(header_list) + '\r\n' - return status_code, status_message, header_data, body + return status_code, status_message, header_data, response_file.read() + class ModuleManager(object): @@ -2243,9 +2739,9 @@ class ModuleManager(object): Path of the module's corresponding Python source file if it exists, or just the module's compiled Python file. If the module has an invalid __file__ attribute, None will be returned. - """ + """ module_file = getattr(module, '__file__', None) - if not module_file or module_file == HardenedModulesHook.EMPTY_MODULE_FILE: + if module_file is None: return None source_file = module_file[:module_file.rfind('py') + 2] @@ -2275,8 +2771,7 @@ class ModuleManager(object): return False def UpdateModuleFileModificationTimes(self): - """Records the current modification times of all monitored modules. - """ + """Records the current modification times of all monitored modules.""" self._modification_times.clear() for name, module in self._modules.items(): if not isinstance(module, types.ModuleType): @@ -2296,6 +2791,10 @@ class ModuleManager(object): self._modules.clear() self._modules.update(self._default_modules) sys.path_hooks[:] = self._save_path_hooks + apiproxy_stub_map.apiproxy.GetPreCallHooks().Clear() + apiproxy_stub_map.apiproxy.GetPostCallHooks().Clear() + + def _ClearTemplateCache(module_dict=sys.modules): @@ -2303,16 +2802,23 @@ def _ClearTemplateCache(module_dict=sys.modules): Attempts to load template module. Ignores failure. If module loads, the template cache is cleared. + + Args: + module_dict: Used for dependency injection. """ template_module = module_dict.get('google.appengine.ext.webapp.template') if template_module is not None: template_module.template_cache.clear() -def CreateRequestHandler(root_path, login_url, require_indexes=False, + +def CreateRequestHandler(root_path, + login_url, + require_indexes=False, static_caching=True): - """Creates a new BaseHTTPRequestHandler sub-class for use with the Python - BaseHTTPServer module's HTTP server. + """Creates a new BaseHTTPRequestHandler sub-class. + + This class will be used with the Python BaseHTTPServer module's HTTP server. Python's built-in HTTP server does not support passing context information along to instances of its request handlers. This function gets around that @@ -2338,9 +2844,11 @@ def CreateRequestHandler(root_path, login_url, require_indexes=False, application_config_cache = AppConfigCache() class DevAppServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): - """Dispatches URLs using patterns from a URLMatcher, which is created by - loading an application's configuration file. Executes CGI scripts in the - local process so the scripts can use mock versions of APIs. + """Dispatches URLs using patterns from a URLMatcher. + + The URLMatcher is created by loading an application's configuration file. + Executes CGI scripts in the local process so the scripts can use mock + versions of APIs. HTTP requests that correctly specify a user info cookie (dev_appserver_login.COOKIE_NAME) will have the 'USER_EMAIL' environment @@ -2359,17 +2867,19 @@ def CreateRequestHandler(root_path, login_url, require_indexes=False, config_cache = application_config_cache + rewriter_chain = CreateResponseRewritersChain() + def __init__(self, *args, **kwargs): """Initializer. Args: - args, kwargs: Positional and keyword arguments passed to the constructor - of the super class. + args: Positional arguments passed to the superclass constructor. + kwargs: Keyword arguments passed to the superclass constructor. """ BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def version_string(self): - """Returns server's version string used for Server HTTP header""" + """Returns server's version string used for Server HTTP header.""" return self.server_version def do_GET(self): @@ -2406,12 +2916,12 @@ def CreateRequestHandler(root_path, login_url, require_indexes=False, server_name = server_name.split(':', 1)[0] env_dict = { - 'REQUEST_METHOD': self.command, - 'REMOTE_ADDR': self.client_address[0], - 'SERVER_SOFTWARE': self.server_version, - 'SERVER_NAME': server_name, - 'SERVER_PROTOCOL': self.protocol_version, - 'SERVER_PORT': str(self.server.server_port), + 'REQUEST_METHOD': self.command, + 'REMOTE_ADDR': self.client_address[0], + 'SERVER_SOFTWARE': self.server_version, + 'SERVER_NAME': server_name, + 'SERVER_PROTOCOL': self.protocol_version, + 'SERVER_PORT': str(self.server.server_port), } full_url = GetFullURL(server_name, self.server.server_port, self.path) @@ -2432,6 +2942,11 @@ def CreateRequestHandler(root_path, login_url, require_indexes=False, config, explicit_matcher = LoadAppConfig(root_path, self.module_dict, cache=self.config_cache, static_caching=static_caching) + if config.api_version != API_VERSION: + logging.error( + "API versions cannot be switched dynamically: %r != %r", + config.api_version, API_VERSION) + sys.exit(1) env_dict['CURRENT_VERSION_ID'] = config.version + ".1" env_dict['APPLICATION_ID'] = config.application dispatcher = MatcherDispatcher(login_url, @@ -2440,8 +2955,10 @@ def CreateRequestHandler(root_path, login_url, require_indexes=False, if require_indexes: dev_appserver_index.SetupIndexes(config.application, root_path) - infile = cStringIO.StringIO(self.rfile.read( + infile = cStringIO.StringIO() + infile.write(self.rfile.read( int(self.headers.get('content-length', 0)))) + infile.seek(0) request_size = len(infile.getvalue()) if request_size > MAX_REQUEST_SIZE: @@ -2465,7 +2982,8 @@ def CreateRequestHandler(root_path, login_url, require_indexes=False, outfile.flush() outfile.seek(0) - status_code, status_message, header_data, body = RewriteResponse(outfile) + status_code, status_message, header_data, body = ( + RewriteResponse(outfile, self.rewriter_chain)) runtime_response_size = len(outfile.getvalue()) if runtime_response_size > MAX_RUNTIME_RESPONSE_SIZE: @@ -2522,6 +3040,7 @@ def CreateRequestHandler(root_path, login_url, require_indexes=False, return DevAppServerRequestHandler + def ReadAppConfig(appinfo_path, parse_app_config=appinfo.LoadSingleAppInfo): """Reads app.yaml file and returns its app id and list of URLMap instances. @@ -2537,19 +3056,15 @@ def ReadAppConfig(appinfo_path, parse_app_config=appinfo.LoadSingleAppInfo): URLMap instances, this function will raise an InvalidAppConfigError exception. """ - from string import Template try: - appinfo_content = file(appinfo_path, 'r').read() - appinfo_template = Template(appinfo_content) - appinfo_config = appinfo_template.substitute(os.environ) - appinfo_file = cStringIO.StringIO(appinfo_config) - try: - return parse_app_config(appinfo_file) - finally: - appinfo_file.close() - except IOError, e: + appinfo_file = file(appinfo_path, 'r') + except IOError, unused_e: raise InvalidAppConfigError( - 'Application configuration could not be read from "%s"' % appinfo_path) + 'Application configuration could not be read from "%s"' % appinfo_path) + try: + return parse_app_config(appinfo_file) + finally: + appinfo_file.close() def CreateURLMatcherFromMaps(root_path, @@ -2577,17 +3092,28 @@ def CreateURLMatcherFromMaps(root_path, default_expiration: String describing default expiration time for browser based caching of static files. If set to None this disallows any browser caching of static content. - create_url_matcher, create_cgi_dispatcher, create_file_dispatcher, + create_url_matcher: Used for dependency injection. + create_cgi_dispatcher: Used for dependency injection. + create_file_dispatcher: Used for dependency injection. create_path_adjuster: Used for dependency injection. + normpath: Used for dependency injection. Returns: Instance of URLMatcher with the supplied URLMap objects properly loaded. + + Raises: + InvalidAppConfigError: if the handler in url_map_list is an unknown type. """ url_matcher = create_url_matcher() path_adjuster = create_path_adjuster(root_path) cgi_dispatcher = create_cgi_dispatcher(module_dict, root_path, path_adjuster) + static_file_config_matcher = StaticFileConfigMatcher(url_map_list, + path_adjuster, + default_expiration) file_dispatcher = create_file_dispatcher(path_adjuster, - StaticFileConfigMatcher(url_map_list, path_adjuster, default_expiration)) + static_file_config_matcher) + + FakeFile.SetStaticFileConfigMatcher(static_file_config_matcher) for url_map in url_map_list: admin_only = url_map.login == appinfo.LOGIN_ADMIN @@ -2658,10 +3184,14 @@ def LoadAppConfig(root_path, sys.modules dictionary. cache: Instance of AppConfigCache or None. static_caching: True if browser caching of static files should be allowed. - read_app_config, create_matcher: Used for dependency injection. + read_app_config: Used for dependency injection. + create_matcher: Used for dependency injection. Returns: tuple: (AppInfoExternal, URLMatcher) + + Raises: + AppConfigNotFound: if an app.yaml file cannot be found. """ for appinfo_path in [os.path.join(root_path, 'app.yaml'), os.path.join(root_path, 'app.yml')]: @@ -2691,6 +3221,8 @@ def LoadAppConfig(root_path, module_dict, default_expiration) + FakeFile.SetSkippedFiles(config.skip_files) + if cache is not None: cache.path = appinfo_path cache.config = config @@ -2703,17 +3235,47 @@ def LoadAppConfig(root_path, raise AppConfigNotFoundError +def ReadCronConfig(croninfo_path, parse_cron_config=croninfo.LoadSingleCron): + """Reads cron.yaml file and returns a list of CronEntry instances. + + Args: + croninfo_path: String containing the path to the cron.yaml file. + parse_cron_config: Used for dependency injection. + + Returns: + A CronInfoExternal object. + + Raises: + If the config file is unreadable, empty or invalid, this function will + raise an InvalidAppConfigError or a MalformedCronConfiguration exception. + """ + try: + croninfo_file = file(croninfo_path, 'r') + except IOError, e: + raise InvalidAppConfigError( + 'Cron configuration could not be read from "%s": %s' + % (croninfo_path, e)) + try: + return parse_cron_config(croninfo_file) + finally: + croninfo_file.close() + + + def SetupStubs(app_id, **config): """Sets up testing stubs of APIs. Args: app_id: Application ID being served. + config: keyword arguments. Keywords: + root_path: Root path to the directory of the application which should + contain the app.yaml, indexes.yaml, and queues.yaml files. login_url: Relative URL which should be used for handling user login/logout. datastore_path: Path to the file to store Datastore file stub data in. - history_path: Path to the file to store Datastore history in. - clear_datastore: If the datastore and history should be cleared on startup. + history_path: DEPRECATED, No-op. + clear_datastore: If the datastore should be cleared on startup. smtp_host: SMTP host used for sending test mail. smtp_port: SMTP port. smtp_user: SMTP user. @@ -2721,10 +3283,13 @@ def SetupStubs(app_id, **config): enable_sendmail: Whether to use sendmail as an alternative to SMTP. show_mail_body: Whether to log the body of emails. remove: Used for dependency injection. + trusted: True if this app can access data belonging to other apps. This + behavior is different from the real app server and should be left False + except for advanced uses of dev_appserver. """ + root_path = config.get('root_path', None) login_url = config['login_url'] datastore_path = config['datastore_path'] - history_path = config['history_path'] clear_datastore = config['clear_datastore'] require_indexes = config.get('require_indexes', False) smtp_host = config.get('smtp_host', None) @@ -2734,22 +3299,24 @@ def SetupStubs(app_id, **config): enable_sendmail = config.get('enable_sendmail', False) show_mail_body = config.get('show_mail_body', False) remove = config.get('remove', os.remove) + trusted = config.get('trusted', False) os.environ['APPLICATION_ID'] = app_id if clear_datastore: - for path in (datastore_path, history_path): - if os.path.lexists(path): - logging.info('Attempting to remove file at %s', path) - try: - remove(path) - except OSError, e: - logging.warning('Removing file failed: %s', e) + path = datastore_path + if os.path.lexists(path): + logging.info('Attempting to remove file at %s', path) + try: + remove(path) + except OSError, e: + logging.warning('Removing file failed: %s', e) apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() datastore = datastore_file_stub.DatastoreFileStub( - app_id, datastore_path, history_path, require_indexes=require_indexes) + app_id, datastore_path, require_indexes=require_indexes, + trusted=trusted) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore) fixed_login_url = '%s?%s=%%s' % (login_url, @@ -2758,42 +3325,53 @@ def SetupStubs(app_id, **config): dev_appserver_login.LOGOUT_PARAM) apiproxy_stub_map.apiproxy.RegisterStub( - 'user', - user_service_stub.UserServiceStub(login_url=fixed_login_url, - logout_url=fixed_logout_url)) + 'user', + user_service_stub.UserServiceStub(login_url=fixed_login_url, + logout_url=fixed_logout_url)) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'urlfetch', + urlfetch_stub.URLFetchServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( - 'urlfetch', - urlfetch_stub.URLFetchServiceStub()) + 'mail', + mail_stub.MailServiceStub(smtp_host, + smtp_port, + smtp_user, + smtp_password, + enable_sendmail=enable_sendmail, + show_mail_body=show_mail_body)) apiproxy_stub_map.apiproxy.RegisterStub( - 'mail', - mail_stub.MailServiceStub(smtp_host, - smtp_port, - smtp_user, - smtp_password, - enable_sendmail=enable_sendmail, - show_mail_body=show_mail_body)) + 'memcache', + memcache_stub.MemcacheServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( - 'memcache', - memcache_stub.MemcacheServiceStub()) + 'capability_service', + capability_stub.CapabilityServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( - 'capability_service', - capability_stub.CapabilityServiceStub()) + 'taskqueue', + taskqueue_stub.TaskQueueServiceStub(root_path=root_path)) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'xmpp', + xmpp_service_stub.XmppServiceStub()) + + try: from google.appengine.api.images import images_stub apiproxy_stub_map.apiproxy.RegisterStub( - 'images', - images_stub.ImagesServiceStub()) + 'images', + images_stub.ImagesServiceStub()) except ImportError, e: logging.warning('Could not initialize images API; you are likely missing ' 'the Python "PIL" module. ImportError: %s', e) from google.appengine.api.images import images_not_implemented_stub - apiproxy_stub_map.apiproxy.RegisterStub('images', - images_not_implemented_stub.ImagesNotImplementedServiceStub()) + apiproxy_stub_map.apiproxy.RegisterStub( + 'images', + images_not_implemented_stub.ImagesNotImplementedServiceStub()) def CreateImplicitMatcher(module_dict, @@ -2811,7 +3389,9 @@ def CreateImplicitMatcher(module_dict, module_dict: Dictionary in the form used by sys.modules. root_path: Path to the root of the application. login_url: Relative URL which should be used for handling user login/logout. + create_path_adjuster: Used for dependedency injection. create_local_dispatcher: Used for dependency injection. + create_cgi_dispatcher: Used for dependedency injection. Returns: Instance of URLMatcher with appropriate dispatchers. @@ -2827,7 +3407,6 @@ def CreateImplicitMatcher(module_dict, False, False) - admin_dispatcher = create_cgi_dispatcher(module_dict, root_path, path_adjuster) url_matcher.AddURL('/_ah/admin(?:/.*)?', @@ -2871,10 +3450,17 @@ def CreateServer(root_path, template_dir, serve_address='', require_indexes=False, + allow_skipped_files=False, static_caching=True, - python_path_list=sys.path): + python_path_list=sys.path, + sdk_dir=os.path.dirname(os.path.dirname(google.__file__))): """Creates an new HTTPServer for an application. + The sdk_dir argument must be specified for the directory storing all code for + the SDK so as to allow for the sandboxing of module access to work for any + and all SDK code. While typically this is where the 'google' package lives, + it can be in another location because of API version support. + Args: root_path: String containing the path to the root directory of the application where the app.yaml file is. @@ -2884,8 +3470,10 @@ def CreateServer(root_path, are stored. serve_address: Address on which the server should serve. require_indexes: True if index.yaml is read-only gospel; default False. + allow_skipped_files: True if skipped files should be accessible. static_caching: True if browser caching of static files should be allowed. python_path_list: Used for dependency injection. + sdk_dir: Directory where the SDK is stored. Returns: Instance of BaseHTTPServer.HTTPServer that's ready to start accepting. @@ -2893,14 +3481,65 @@ def CreateServer(root_path, absolute_root_path = os.path.realpath(root_path) SetupTemplates(template_dir) - FakeFile.SetAllowedPaths([absolute_root_path, - os.path.dirname(os.path.dirname(google.__file__)), + FakeFile.SetAllowedPaths(absolute_root_path, + [sdk_dir, template_dir]) + FakeFile.SetAllowSkippedFiles(allow_skipped_files) - handler_class = CreateRequestHandler(absolute_root_path, login_url, - require_indexes, static_caching) + handler_class = CreateRequestHandler(absolute_root_path, + login_url, + require_indexes, + static_caching) if absolute_root_path not in python_path_list: python_path_list.insert(0, absolute_root_path) + return HTTPServerWithScheduler((serve_address, port), handler_class) + + +class HTTPServerWithScheduler(BaseHTTPServer.HTTPServer): + """A BaseHTTPServer subclass that calls a method at a regular interval.""" + + def __init__(self, server_address, request_handler_class): + """Constructor. + + Args: + server_address: the bind address of the server. + request_handler_class: class used to handle requests. + """ + BaseHTTPServer.HTTPServer.__init__(self, server_address, + request_handler_class) + self._events = [] - return BaseHTTPServer.HTTPServer((serve_address, port), handler_class) + def get_request(self, time_func=time.time, select_func=select.select): + """Overrides the base get_request call. + + Args: + time_func: used for testing. + select_func: used for testing. + + Returns: + a (socket_object, address info) tuple. + """ + while True: + if self._events: + current_time = time_func() + next_eta = self._events[0][0] + delay = next_eta - current_time + else: + delay = DEFAULT_SELECT_DELAY + readable, _, _ = select_func([self.socket], [], [], max(delay, 0)) + if readable: + return self.socket.accept() + current_time = time_func() + if self._events and current_time >= self._events[0][0]: + unused_eta, runnable = heapq.heappop(self._events) + runnable() + + def AddEvent(self, eta, runnable): + """Add a runnable event to be run at the specified time. + + Args: + eta: when to run the event, in seconds since epoch. + runnable: a callable object. + """ + heapq.heappush(self._events, (eta, runnable)) google_appengine/google/appengine/tools/dev_appserver.py @@ -76,13 +76,6 @@ def GenerateIndexFromHistory(query_history, res = [] for (kind, ancestor, props), count in sorted(indexes.iteritems()): res.append('') - if count == 0: - message = '# Unused in query history -- copied from input.' - elif count == 1: - message = '# Used once in query history.' - else: - message = '# Used %d times in query history.' % count - res.append(message) res.append(datastore_index.IndexYamlForQuery(kind, ancestor, props)) res.append('') google_appengine/google/appengine/tools/dev_appserver_index.py @@ -28,8 +28,9 @@ supply no parameters. """ -import Cookie import cgi +import Cookie +import md5 import os import sys import urllib @@ -66,8 +67,8 @@ def GetUserInfo(http_cookie, cookie_name=COOKIE_NAME): if cookie_name in cookie: cookie_value = cookie[cookie_name].value - email, admin = (cookie_value.split(':') + ['', ''])[:2] - return email, (admin == 'True') + email, admin, user_id = (cookie_value.split(':') + ['', '', ''])[:3] + return email, (admin == 'True'), user_id def CreateCookieData(email, admin): @@ -82,7 +83,12 @@ def CreateCookieData(email, admin): admin_string = 'False' if admin: admin_string = 'True' - return '%s:%s' % (email, admin_string) + if email: + user_id_digest = md5.new(email.lower()).digest() + user_id = '1' + ''.join(['%02d' % ord(x) for x in user_id_digest])[:20] + else: + user_id = '' + return '%s:%s:%s' % (email, admin_string, user_id) def SetUserInfoCookie(email, admin, cookie_name=COOKIE_NAME): google_appengine/google/appengine/tools/dev_appserver_login.py @@ -54,6 +54,8 @@ Options: --debug_imports Enables debug logging for module imports, showing search paths used for finding modules and any errors encountered during the import process. + --allow_skipped_files Allow access to files matched by app.yaml's + skipped_files (default False) --disable_static_caching Never allow the browser to cache static files. (Default enable if expiration set in app.yaml) """ @@ -65,14 +67,30 @@ from google.appengine.tools import os_compat import getopt import logging import os +import re +import signal import sys import traceback import tempfile -from google.appengine.api import yaml_errors -from google.appengine.tools import appcfg -from google.appengine.tools import appengine_rpc -from google.appengine.tools import dev_appserver +logging.basicConfig( + level=logging.INFO, + format='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s') + + +def SetGlobals(): + """Set various global variables involving the 'google' package. + + This function should not be called until sys.path has been properly set. + """ + global yaml_errors, appcfg, appengine_rpc, dev_appserver, os_compat + from google.appengine.api import yaml_errors + from google.appengine.dist import py_zipimport + from google.appengine.tools import appcfg + from google.appengine.tools import appengine_rpc + from google.appengine.tools import dev_appserver + from google.appengine.tools import os_compat + DEFAULT_ADMIN_CONSOLE_SERVER = 'appengine.google.com' @@ -91,16 +109,22 @@ ARG_LOGIN_URL = 'login_url' ARG_LOG_LEVEL = 'log_level' ARG_PORT = 'port' ARG_REQUIRE_INDEXES = 'require_indexes' +ARG_ALLOW_SKIPPED_FILES = 'allow_skipped_files' ARG_SMTP_HOST = 'smtp_host' ARG_SMTP_PASSWORD = 'smtp_password' ARG_SMTP_PORT = 'smtp_port' ARG_SMTP_USER = 'smtp_user' ARG_STATIC_CACHING = 'static_caching' ARG_TEMPLATE_DIR = 'template_dir' +ARG_TRUSTED = 'trusted' - -BASE_PATH = os.path.abspath( - os.path.join(os.path.dirname(dev_appserver.__file__), '../../../')) +SDK_PATH = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os_compat.__file__) + ) + ) + ) DEFAULT_ARGS = { ARG_PORT: 8080, @@ -112,7 +136,7 @@ DEFAULT_ARGS = { ARG_LOGIN_URL: '/_ah/login', ARG_CLEAR_DATASTORE: False, ARG_REQUIRE_INDEXES: False, - ARG_TEMPLATE_DIR: os.path.join(BASE_PATH, 'templates'), + ARG_TEMPLATE_DIR: os.path.join(SDK_PATH, 'templates'), ARG_SMTP_HOST: '', ARG_SMTP_PORT: 25, ARG_SMTP_USER: '', @@ -123,9 +147,79 @@ DEFAULT_ARGS = { ARG_ADDRESS: 'localhost', ARG_ADMIN_CONSOLE_SERVER: DEFAULT_ADMIN_CONSOLE_SERVER, ARG_ADMIN_CONSOLE_HOST: None, + ARG_ALLOW_SKIPPED_FILES: False, ARG_STATIC_CACHING: True, + ARG_TRUSTED: False, } +API_PATHS = {'1': + {'google': (), + 'antlr3': ('lib', 'antlr3'), + 'django': ('lib', 'django'), + 'webob': ('lib', 'webob'), + 'yaml': ('lib', 'yaml', 'lib'), + } + } + +DEFAULT_API_VERSION = '1' + +API_PATHS['test'] = API_PATHS[DEFAULT_API_VERSION].copy() +API_PATHS['test']['_test'] = ('nonexistent', 'test', 'path') + + +def SetPaths(app_config_path): + """Set the interpreter to use the specified API version. + + The app.yaml file is scanned for the api_version field and the value is + extracted. With that information, the paths in API_PATHS are added to the + front of sys.paths to make sure that they take precedent over any other paths + to older versions of a package. All modules for each package set are cleared + out of sys.modules to make sure only the newest version is used. + + Args: + - app_config_path: Path to the app.yaml file. + """ + api_version_re = re.compile(r'api_version:\s*(?P<api_version>[\w.]{1,32})') + api_version = None + app_config_file = open(app_config_path, 'r') + try: + for line in app_config_file: + re_match = api_version_re.match(line) + if re_match: + api_version = re_match.group('api_version') + break + finally: + app_config_file.close() + + if api_version is None: + logging.error("Application configuration file missing an 'api_version' " + "value:\n%s" % app_config_path) + sys.exit(1) + if api_version not in API_PATHS: + logging.error("Value of %r for 'api_version' from the application " + "configuration file is not valid:\n%s" % + (api_version, app_config_path)) + sys.exit(1) + + if api_version == DEFAULT_API_VERSION: + return DEFAULT_API_VERSION + + sdk_path = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os_compat.__file__) + ) + ) + ) + for pkg_name, path_parts in API_PATHS[api_version].iteritems(): + for name in sys.modules.keys(): + if name == pkg_name or name.startswith('%s.' % pkg_name): + del sys.modules[name] + pkg_path = os.path.join(sdk_path, *path_parts) + sys.path.insert(0, pkg_path) + + return api_version + def PrintUsageExit(code): """Prints usage information and exits with a status code. @@ -163,6 +257,7 @@ def ParseArguments(argv): [ 'address=', 'admin_console_server=', 'admin_console_host=', + 'allow_skipped_files', 'auth_domain=', 'clear_datastore', 'datastore_path=', @@ -180,6 +275,7 @@ def ParseArguments(argv): 'smtp_port=', 'smtp_user=', 'template_dir=', + 'trusted', ]) except getopt.GetoptError, e: print >>sys.stderr, 'Error: %s' % e @@ -205,10 +301,10 @@ def ParseArguments(argv): option_dict[ARG_ADDRESS] = value if option == '--datastore_path': - option_dict[ARG_DATASTORE_PATH] = value + option_dict[ARG_DATASTORE_PATH] = os.path.abspath(value) if option == '--history_path': - option_dict[ARG_HISTORY_PATH] = value + option_dict[ARG_HISTORY_PATH] = os.path.abspath(value) if option in ('-c', '--clear_datastore'): option_dict[ARG_CLEAR_DATASTORE] = True @@ -241,10 +337,10 @@ def ParseArguments(argv): option_dict[ARG_SHOW_MAIL_BODY] = True if option == '--auth_domain': - dev_appserver.DEFAULT_ENV['AUTH_DOMAIN'] = value + option_dict['_DEFAULT_ENV_AUTH_DOMAIN'] = value if option == '--debug_imports': - dev_appserver.HardenedModulesHook.ENABLE_LOGGING = True + option_dict['_ENABLE_LOGGING'] = True if option == '--template_dir': option_dict[ARG_TEMPLATE_DIR] = value @@ -255,9 +351,15 @@ def ParseArguments(argv): if option == '--admin_console_host': option_dict[ARG_ADMIN_CONSOLE_HOST] = value + if option == '--allow_skipped_files': + option_dict[ARG_ALLOW_SKIPPED_FILES] = True + if option == '--disable_static_caching': option_dict[ARG_STATIC_CACHING] = False + if option == '--trusted': + option_dict[ARG_TRUSTED] = True + return args, option_dict @@ -282,6 +384,14 @@ def MakeRpcServer(option_dict): return server +def SigTermHandler(signum, frame): + """Handler for TERM signal. + + Raises a KeyboardInterrupt to perform a graceful shutdown on SIGTERM signal. + """ + raise KeyboardInterrupt() + + def main(argv): """Runs the development application server.""" args, option_dict = ParseArguments(argv) @@ -291,6 +401,25 @@ def main(argv): PrintUsageExit(1) root_path = args[0] + for suffix in ('yaml', 'yml'): + path = os.path.join(root_path, 'app.%s' % suffix) + if os.path.exists(path): + api_version = SetPaths(path) + break + else: + logging.error("Application configuration file not found in %s" % root_path) + return 1 + + SetGlobals() + dev_appserver.API_VERSION = api_version + + if '_DEFAULT_ENV_AUTH_DOMAIN' in option_dict: + auth_domain = option_dict['_DEFAULT_ENV_AUTH_DOMAIN'] + dev_appserver.DEFAULT_ENV['AUTH_DOMAIN'] = auth_domain + if '_ENABLE_LOGGING' in option_dict: + enable_logging = option_dict['_ENABLE_LOGGING'] + dev_appserver.HardenedModulesHook.ENABLE_LOGGING = enable_logging + log_level = option_dict[ARG_LOG_LEVEL] port = option_dict[ARG_PORT] datastore_path = option_dict[ARG_DATASTORE_PATH] @@ -298,11 +427,12 @@ def main(argv): template_dir = option_dict[ARG_TEMPLATE_DIR] serve_address = option_dict[ARG_ADDRESS] require_indexes = option_dict[ARG_REQUIRE_INDEXES] + allow_skipped_files = option_dict[ARG_ALLOW_SKIPPED_FILES] static_caching = option_dict[ARG_STATIC_CACHING] - logging.basicConfig( - level=log_level, - format='%(levelname)-8s %(asctime)s %(filename)s] %(message)s') + option_dict['root_path'] = os.path.realpath(root_path) + + logging.getLogger().setLevel(log_level) config = None try: @@ -331,13 +461,18 @@ def main(argv): exc_type, exc_value, exc_traceback))) return 1 - http_server = dev_appserver.CreateServer(root_path, - login_url, - port, - template_dir, - serve_address=serve_address, - require_indexes=require_indexes, - static_caching=static_caching) + http_server = dev_appserver.CreateServer( + root_path, + login_url, + port, + template_dir, + sdk_dir=SDK_PATH, + serve_address=serve_address, + require_indexes=require_indexes, + allow_skipped_files=allow_skipped_files, + static_caching=static_caching) + + signal.signal(signal.SIGTERM, SigTermHandler) logging.info('Running application %s on port %d: http://%s:%d', config.application, port, serve_address, port) @@ -359,3 +494,5 @@ def main(argv): if __name__ == '__main__': sys.exit(main(sys.argv)) +else: + SetGlobals() google_appengine/google/appengine/tools/dev_appserver_main.py @@ -42,3 +42,5 @@ else: ERROR_PATH_NOT_FOUND = 3 +ERROR_ACCESS_DENIED = 5 +ERROR_ALREADY_EXISTS = 183 \ No newline at end of file google_appengine/google/appengine/tools/os_compat.py @@ -214,7 +214,7 @@ class ProtocolMessage: return self.DebugFormatFixed32(value) return "%d" % value def DebugFormatInt64(self, value): - if (value <= -2000000000 or value >= 2000000000): + if (value <= -20000000000000 or value >= 20000000000000): return self.DebugFormatFixed64(value) return "%d" % value def DebugFormatString(self, value): @@ -344,13 +344,13 @@ class Encoder: def putFloat(self, v): a = array.array('B') - a.fromstring(struct.pack("f", v)) + a.fromstring(struct.pack("<f", v)) self.buf.extend(a) return def putDouble(self, v): a = array.array('B') - a.fromstring(struct.pack("d", v)) + a.fromstring(struct.pack("<d", v)) self.buf.extend(a) return @@ -362,6 +362,7 @@ class Encoder: return def putPrefixedString(self, v): + v = str(v) self.putVarInt32(len(v)) self.buf.fromstring(v) return @@ -499,13 +500,13 @@ class Decoder: if self.idx + 4 > self.limit: raise ProtocolBufferDecodeError, "truncated" a = self.buf[self.idx:self.idx+4] self.idx += 4 - return struct.unpack("f", a)[0] + return struct.unpack("<f", a)[0] def getDouble(self): if self.idx + 8 > self.limit: raise ProtocolBufferDecodeError, "truncated" a = self.buf[self.idx:self.idx+8] self.idx += 8 - return struct.unpack("d", a)[0] + return struct.unpack("<d", a)[0] def getBoolean(self): b = self.get8() google_appengine/google/net/proto/ProtocolBuffer.py @@ -138,7 +138,7 @@ bug in your grammar, it can only be detected at runtime. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = '3.1' +__version__ = '3.1.1' def version_str_to_tuple(version_str): import re google_appengine/lib/antlr3/antlr3/__init__.py @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: antlr-python-runtime -Version: 3.1 +Version: 3.1.1 Summary: Runtime package for ANTLR3 Home-page: http://www.antlr.org/ Author: Benjamin Niemann google_appengine/lib/antlr3/antlr_python_runtime.egg-info/PKG-INFO @@ -267,7 +267,7 @@ class functest(Command): setup(name='antlr_python_runtime', - version='3.1', + version='3.1.1', packages=['antlr3'], author="Benjamin Niemann", google_appengine/lib/antlr3/setup.py @@ -1 +1 @@ -VERSION = (0, 96.1, None) +VERSION = (0, 96.4, None) google_appengine/lib/django/django/__init__.py @@ -237,7 +237,7 @@ TRANSACTIONS_MANAGED = False # The User-Agent string to use when checking for URL validity through the # isExistingURL validator. -URL_VALIDATOR_USER_AGENT = "Django/0.96.1 (http://www.djangoproject.com)" +URL_VALIDATOR_USER_AGENT = "Django/0.96.2 (http://www.djangoproject.com)" ############## # MIDDLEWARE # google_appengine/lib/django/django/conf/global_settings.py @@ -19,7 +19,6 @@ <div class="form-row"> <label for="id_password">{% trans 'Password:' %}</label> <input type="password" name="password" id="id_password" /> <input type="hidden" name="this_is_the_login_form" value="1" /> - <input type="hidden" name="post_data" value="{{ post_data }}" /> {#<span class="help">{% trans 'Have you <a href="/password_reset/">forgotten your password</a>?' %}</span>#} </div> <div class="submit-row"> <label>&nbsp;</label><input type="submit" value="{% trans 'Log in' %}" /> google_appengine/lib/django/django/contrib/admin/templates/admin/login.html @@ -3,43 +3,21 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.auth import authenticate, login from django.shortcuts import render_to_response +from django.utils.html import escape from django.utils.translation import gettext_lazy -import base64, datetime, md5 -import cPickle as pickle +import base64, datetime ERROR_MESSAGE = gettext_lazy("Please enter a correct username and password. Note that both fields are case-sensitive.") LOGIN_FORM_KEY = 'this_is_the_login_form' def _display_login_form(request, error_message=''): request.session.set_test_cookie() - if request.POST and request.POST.has_key('post_data'): - # User has failed login BUT has previously saved post data. - post_data = request.POST['post_data'] - elif request.POST: - # User's session must have expired; save their post data. - post_data = _encode_post_data(request.POST) - else: - post_data = _encode_post_data({}) return render_to_response('admin/login.html', { 'title': _('Log in'), - 'app_path': request.path, - 'post_data': post_data, + 'app_path': escape(request.path), 'error_message': error_message }, context_instance=template.RequestContext(request)) -def _encode_post_data(post_data): - pickled = pickle.dumps(post_data) - pickled_md5 = md5.new(pickled + settings.SECRET_KEY).hexdigest() - return base64.encodestring(pickled + pickled_md5) - -def _decode_post_data(encoded_data): - encoded_data = base64.decodestring(encoded_data) - pickled, tamper_check = encoded_data[:-32], encoded_data[-32:] - if md5.new(pickled + settings.SECRET_KEY).hexdigest() != tamper_check: - from django.core.exceptions import SuspiciousOperation - raise SuspiciousOperation, "User may have tampered with session cookie." - return pickle.loads(pickled) - def staff_member_required(view_func): """ Decorator for views that checks that the user is logged in and is a staff @@ -48,10 +26,6 @@ def staff_member_required(view_func): def _checklogin(request, *args, **kwargs): if request.user.is_authenticated() and request.user.is_staff: # The user is valid. Continue to the admin page. - if request.POST.has_key('post_data'): - # User must have re-authenticated through a different window - # or tab. - request.POST = _decode_post_data(request.POST['post_data']) return view_func(request, *args, **kwargs) assert hasattr(request, 'session'), "The Django admin requires session middleware to be installed. Edit your MIDDLEWARE_CLASSES setting to insert 'django.contrib.sessions.middleware.SessionMiddleware'." @@ -59,7 +33,7 @@ def staff_member_required(view_func): # If this isn't already the login page, display it. if not request.POST.has_key(LOGIN_FORM_KEY): if request.POST: - message = _("Please log in again, because your session has expired. Don't worry: Your submission has been saved.") + message = _("Please log in again, because your session has expired.") else: message = "" return _display_login_form(request, message) @@ -92,16 +66,7 @@ def staff_member_required(view_func): # TODO: set last_login with an event. user.last_login = datetime.datetime.now() user.save() - if request.POST.has_key('post_data'): - post_data = _decode_post_data(request.POST['post_data']) - if post_data and not post_data.has_key(LOGIN_FORM_KEY): - # overwrite request.POST with the saved post_data, and continue - request.POST = post_data - request.user = user - return view_func(request, *args, **kwargs) - else: - request.session.delete_test_cookie() - return http.HttpResponseRedirect(request.path) + return http.HttpResponseRedirect(request.path) else: return _display_login_form(request, ERROR_MESSAGE) google_appengine/lib/django/django/contrib/admin/views/decorators.py @@ -1192,9 +1192,7 @@ def runserver(addr, port, use_reloader=True, admin_media_dir=''): print "Development server is running at http://%s:%s/" % (addr, port) print "Quit the server with %s." % quit_command try: - import django - path = admin_media_dir or django.__path__[0] + '/contrib/admin/media' - handler = AdminMediaHandler(WSGIHandler(), path) + handler = AdminMediaHandler(WSGIHandler(), admin_media_path) run(addr, int(port), handler) except WSGIServerException, e: # Use helpful error messages instead of ugly tracebacks. google_appengine/lib/django/django/core/management.py @@ -11,6 +11,8 @@ from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from types import ListType, StringType import os, re, sys, time, urllib +from django.utils._os import safe_join + __version__ = "0.1" __all__ = ['WSGIServer','WSGIRequestHandler','demo_app'] @@ -599,11 +601,25 @@ class AdminMediaHandler(object): self.application = application if not media_dir: import django - self.media_dir = django.__path__[0] + '/contrib/admin/media' + self.media_dir = \ + os.path.join(django.__path__[0], 'contrib', 'admin', 'media') else: self.media_dir = media_dir self.media_url = settings.ADMIN_MEDIA_PREFIX + def file_path(self, url): + """ + Returns the path to the media file on disk for the given URL. + + The passed URL is assumed to begin with ADMIN_MEDIA_PREFIX. If the + resultant file path is outside the media directory, then a ValueError + is raised. + """ + # Remove ADMIN_MEDIA_PREFIX. + relative_url = url[len(self.media_url):] + relative_path = urllib.url2pathname(relative_url) + return safe_join(self.media_dir, relative_path) + def __call__(self, environ, start_response): import os.path @@ -614,19 +630,25 @@ class AdminMediaHandler(object): return self.application(environ, start_response) # Find the admin file and serve it up, if it exists and is readable. - relative_url = environ['PATH_INFO'][len(self.media_url):] - file_path = os.path.join(self.media_dir, relative_url) + try: + file_path = self.file_path(environ['PATH_INFO']) + except ValueError: # Resulting file path was not valid. + status = '404 NOT FOUND' + headers = {'Content-type': 'text/plain'} + output = ['Page not found: %s' % environ['PATH_INFO']] + start_response(status, headers.items()) + return output if not os.path.exists(file_path): status = '404 NOT FOUND' headers = {'Content-type': 'text/plain'} - output = ['Page not found: %s' % file_path] + output = ['Page not found: %s' % environ['PATH_INFO']] else: try: fp = open(file_path, 'rb') except IOError: status = '401 UNAUTHORIZED' headers = {'Content-type': 'text/plain'} - output = ['Permission denied: %s' % file_path] + output = ['Permission denied: %s' % environ['PATH_INFO']] else: status = '200 OK' headers = {} google_appengine/lib/django/django/core/servers/basehttp.py @@ -34,10 +34,11 @@ if len(sys.argv) > 1 and sys.argv[1] == 'bdist_wininst': setup( name = "Django", - version = "0.96.1", + version = "0.96.4", url = 'http://www.djangoproject.com/', - author = 'Lawrence Journal-World', - author_email = 'holovaty@gmail.com', + author = 'Django Software Foundation', + author_email = 'foundation@djangoproject.com', + download_url = 'http://media.djangoproject.com/releases/0.96/Django-0.96.4.tar.gz', description = 'A high-level Python Web framework that encourages rapid development and clean, pragmatic design.', packages = packages, data_files = data_files, google_appengine/lib/django/setup.py google_appengine/lib/django/MANIFEST.in google_appengine/lib/django/django/contrib/flatpages/README.TXT google_appengine/lib/django/django/contrib/formtools/templates/formtools/form.html google_appengine/lib/django/django/contrib/formtools/templates/formtools/preview.html google_appengine/lib/django/django/contrib/redirects/README.TXT google_appengine/lib/django/django/dispatch/license.txt google_appengine/lib/django/django/utils/simplejson/LICENSE.txt google_appengine/lib/django/examples/__init__.py google_appengine/lib/django/examples/hello/__init__.py google_appengine/lib/django/examples/hello/urls.py google_appengine/lib/django/examples/hello/views.py google_appengine/lib/django/examples/manage.py google_appengine/lib/django/examples/settings.py google_appengine/lib/django/examples/urls.py google_appengine/lib/django/examples/views.py google_appengine/lib/django/extras/README.TXT google_appengine/lib/django/extras/django_bash_completion google_appengine/lib/django/tests/modeltests/__init__.py google_appengine/lib/django/tests/modeltests/basic/__init__.py google_appengine/lib/django/tests/modeltests/basic/models.py google_appengine/lib/django/tests/modeltests/choices/__init__.py google_appengine/lib/django/tests/modeltests/choices/models.py google_appengine/lib/django/tests/modeltests/custom_columns/__init__.py google_appengine/lib/django/tests/modeltests/custom_columns/models.py google_appengine/lib/django/tests/modeltests/custom_managers/__init__.py google_appengine/lib/django/tests/modeltests/custom_managers/models.py google_appengine/lib/django/tests/modeltests/custom_methods/__init__.py google_appengine/lib/django/tests/modeltests/custom_methods/models.py google_appengine/lib/django/tests/modeltests/custom_pk/__init__.py google_appengine/lib/django/tests/modeltests/custom_pk/models.py google_appengine/lib/django/tests/modeltests/empty/__init__.py google_appengine/lib/django/tests/modeltests/empty/models.py google_appengine/lib/django/tests/modeltests/field_defaults/__init__.py google_appengine/lib/django/tests/modeltests/field_defaults/models.py google_appengine/lib/django/tests/modeltests/fixtures/__init__.py google_appengine/lib/django/tests/modeltests/fixtures/fixtures/fixture1.json google_appengine/lib/django/tests/modeltests/fixtures/fixtures/fixture2.json google_appengine/lib/django/tests/modeltests/fixtures/fixtures/fixture2.xml google_appengine/lib/django/tests/modeltests/fixtures/fixtures/fixture3.xml google_appengine/lib/django/tests/modeltests/fixtures/fixtures/initial_data.json google_appengine/lib/django/tests/modeltests/fixtures/models.py google_appengine/lib/django/tests/modeltests/generic_relations/__init__.py google_appengine/lib/django/tests/modeltests/generic_relations/models.py google_appengine/lib/django/tests/modeltests/get_latest/__init__.py google_appengine/lib/django/tests/modeltests/get_latest/models.py google_appengine/lib/django/tests/modeltests/get_object_or_404/__init__.py google_appengine/lib/django/tests/modeltests/get_object_or_404/models.py google_appengine/lib/django/tests/modeltests/get_or_create/__init__.py google_appengine/lib/django/tests/modeltests/get_or_create/models.py google_appengine/lib/django/tests/modeltests/invalid_models/__init__.py google_appengine/lib/django/tests/modeltests/invalid_models/models.py google_appengine/lib/django/tests/modeltests/lookup/__init__.py google_appengine/lib/django/tests/modeltests/lookup/models.py google_appengine/lib/django/tests/modeltests/m2m_and_m2o/__init__.py google_appengine/lib/django/tests/modeltests/m2m_and_m2o/models.py google_appengine/lib/django/tests/modeltests/m2m_intermediary/__init__.py google_appengine/lib/django/tests/modeltests/m2m_intermediary/models.py google_appengine/lib/django/tests/modeltests/m2m_multiple/__init__.py google_appengine/lib/django/tests/modeltests/m2m_multiple/models.py google_appengine/lib/django/tests/modeltests/m2m_recursive/__init__.py google_appengine/lib/django/tests/modeltests/m2m_recursive/models.py google_appengine/lib/django/tests/modeltests/m2o_recursive/__init__.py google_appengine/lib/django/tests/modeltests/m2o_recursive/models.py google_appengine/lib/django/tests/modeltests/m2o_recursive2/__init__.py google_appengine/lib/django/tests/modeltests/m2o_recursive2/models.py google_appengine/lib/django/tests/modeltests/manipulators/__init__.py google_appengine/lib/django/tests/modeltests/manipulators/models.py google_appengine/lib/django/tests/modeltests/many_to_many/__init__.py google_appengine/lib/django/tests/modeltests/many_to_many/models.py google_appengine/lib/django/tests/modeltests/many_to_one/__init__.py google_appengine/lib/django/tests/modeltests/many_to_one/models.py google_appengine/lib/django/tests/modeltests/many_to_one_null/__init__.py google_appengine/lib/django/tests/modeltests/many_to_one_null/models.py google_appengine/lib/django/tests/modeltests/model_forms/__init__.py google_appengine/lib/django/tests/modeltests/model_forms/models.py google_appengine/lib/django/tests/modeltests/model_inheritance/__init__.py google_appengine/lib/django/tests/modeltests/model_inheritance/models.py google_appengine/lib/django/tests/modeltests/mutually_referential/__init__.py google_appengine/lib/django/tests/modeltests/mutually_referential/models.py google_appengine/lib/django/tests/modeltests/one_to_one/__init__.py google_appengine/lib/django/tests/modeltests/one_to_one/models.py google_appengine/lib/django/tests/modeltests/or_lookups/__init__.py google_appengine/lib/django/tests/modeltests/or_lookups/models.py google_appengine/lib/django/tests/modeltests/ordering/__init__.py google_appengine/lib/django/tests/modeltests/ordering/models.py google_appengine/lib/django/tests/modeltests/pagination/__init__.py google_appengine/lib/django/tests/modeltests/pagination/models.py google_appengine/lib/django/tests/modeltests/properties/__init__.py google_appengine/lib/django/tests/modeltests/properties/models.py google_appengine/lib/django/tests/modeltests/reserved_names/__init__.py google_appengine/lib/django/tests/modeltests/reserved_names/models.py google_appengine/lib/django/tests/modeltests/reverse_lookup/__init__.py google_appengine/lib/django/tests/modeltests/reverse_lookup/models.py google_appengine/lib/django/tests/modeltests/save_delete_hooks/__init__.py google_appengine/lib/django/tests/modeltests/save_delete_hooks/models.py google_appengine/lib/django/tests/modeltests/select_related/__init__.py google_appengine/lib/django/tests/modeltests/select_related/models.py google_appengine/lib/django/tests/modeltests/serializers/__init__.py google_appengine/lib/django/tests/modeltests/serializers/models.py google_appengine/lib/django/tests/modeltests/str/__init__.py google_appengine/lib/django/tests/modeltests/str/models.py google_appengine/lib/django/tests/modeltests/test_client/__init__.py google_appengine/lib/django/tests/modeltests/test_client/fixtures/testdata.json google_appengine/lib/django/tests/modeltests/test_client/models.py google_appengine/lib/django/tests/modeltests/test_client/urls.py google_appengine/lib/django/tests/modeltests/test_client/views.py google_appengine/lib/django/tests/modeltests/transactions/__init__.py google_appengine/lib/django/tests/modeltests/transactions/models.py google_appengine/lib/django/tests/modeltests/validation/__init__.py google_appengine/lib/django/tests/modeltests/validation/models.py google_appengine/lib/django/tests/regressiontests/__init__.py google_appengine/lib/django/tests/regressiontests/bug639/__init__.py google_appengine/lib/django/tests/regressiontests/bug639/models.py google_appengine/lib/django/tests/regressiontests/bug639/test.jpg google_appengine/lib/django/tests/regressiontests/bug639/tests.py google_appengine/lib/django/tests/regressiontests/cache/__init__.py google_appengine/lib/django/tests/regressiontests/cache/models.py google_appengine/lib/django/tests/regressiontests/cache/tests.py google_appengine/lib/django/tests/regressiontests/datastructures/__init__.py google_appengine/lib/django/tests/regressiontests/datastructures/models.py google_appengine/lib/django/tests/regressiontests/datastructures/tests.py google_appengine/lib/django/tests/regressiontests/dateformat/__init__.py google_appengine/lib/django/tests/regressiontests/dateformat/models.py google_appengine/lib/django/tests/regressiontests/dateformat/tests.py google_appengine/lib/django/tests/regressiontests/db_typecasts/__init__.py google_appengine/lib/django/tests/regressiontests/db_typecasts/models.py google_appengine/lib/django/tests/regressiontests/db_typecasts/tests.py google_appengine/lib/django/tests/regressiontests/defaultfilters/__init__.py google_appengine/lib/django/tests/regressiontests/defaultfilters/models.py google_appengine/lib/django/tests/regressiontests/defaultfilters/tests.py google_appengine/lib/django/tests/regressiontests/dispatch/__init__.py google_appengine/lib/django/tests/regressiontests/dispatch/models.py google_appengine/lib/django/tests/regressiontests/dispatch/tests/__init__.py google_appengine/lib/django/tests/regressiontests/dispatch/tests/test_dispatcher.py google_appengine/lib/django/tests/regressiontests/dispatch/tests/test_robustapply.py google_appengine/lib/django/tests/regressiontests/dispatch/tests/test_saferef.py google_appengine/lib/django/tests/regressiontests/forms/__init__.py google_appengine/lib/django/tests/regressiontests/forms/models.py google_appengine/lib/django/tests/regressiontests/forms/tests.py google_appengine/lib/django/tests/regressiontests/httpwrappers/__init__.py google_appengine/lib/django/tests/regressiontests/httpwrappers/models.py google_appengine/lib/django/tests/regressiontests/httpwrappers/tests.py google_appengine/lib/django/tests/regressiontests/humanize/__init__.py google_appengine/lib/django/tests/regressiontests/humanize/models.py google_appengine/lib/django/tests/regressiontests/humanize/tests.py google_appengine/lib/django/tests/regressiontests/initial_sql_regress/__init__.py google_appengine/lib/django/tests/regressiontests/initial_sql_regress/models.py google_appengine/lib/django/tests/regressiontests/initial_sql_regress/sql/simple.sql google_appengine/lib/django/tests/regressiontests/invalid_admin_options/__init__.py google_appengine/lib/django/tests/regressiontests/invalid_admin_options/models.py google_appengine/lib/django/tests/regressiontests/many_to_one_regress/__init__.py google_appengine/lib/django/tests/regressiontests/many_to_one_regress/models.py google_appengine/lib/django/tests/regressiontests/markup/__init__.py google_appengine/lib/django/tests/regressiontests/markup/models.py google_appengine/lib/django/tests/regressiontests/markup/tests.py google_appengine/lib/django/tests/regressiontests/null_queries/__init__.py google_appengine/lib/django/tests/regressiontests/null_queries/models.py google_appengine/lib/django/tests/regressiontests/one_to_one_regress/__init__.py google_appengine/lib/django/tests/regressiontests/one_to_one_regress/models.py google_appengine/lib/django/tests/regressiontests/serializers_regress/__init__.py google_appengine/lib/django/tests/regressiontests/serializers_regress/models.py google_appengine/lib/django/tests/regressiontests/serializers_regress/tests.py google_appengine/lib/django/tests/regressiontests/string_lookup/__init__.py google_appengine/lib/django/tests/regressiontests/string_lookup/models.py google_appengine/lib/django/tests/regressiontests/templates/__init__.py google_appengine/lib/django/tests/regressiontests/templates/models.py google_appengine/lib/django/tests/regressiontests/templates/tests.py google_appengine/lib/django/tests/regressiontests/templates/urls.py google_appengine/lib/django/tests/regressiontests/templates/views.py google_appengine/lib/django/tests/regressiontests/urlpatterns_reverse/__init__.py google_appengine/lib/django/tests/regressiontests/urlpatterns_reverse/models.py google_appengine/lib/django/tests/regressiontests/urlpatterns_reverse/tests.py google_appengine/lib/django/tests/runtests.py google_appengine/lib/django/tests/templates/404.html google_appengine/lib/django/tests/templates/500.html google_appengine/lib/django/tests/templates/login.html google_appengine/lib/django/tests/urls.py d06c0e266c629004441426e22c61e76a0b88183c Jason Smith jhs@proven-corporation.com http://github.com/jhs/app-engine-console/commit/a4234a2c08402581f3bba8a014dc61073ba95e93 a4234a2c08402581f3bba8a014dc61073ba95e93 2009-10-14T03:10:50-07:00 2009-10-14T03:10:50-07:00 Upgrade to SDK version 1.2.6 1d518b3f1e860e403da28f560ba86ba5dc9dad76 Jason Smith jhs@proven-corporation.com