From 2f16d99dada72b8ac0c07f3b557134df9160502e Mon Sep 17 00:00:00 2001 From: Benjamin Kaplan Date: Mon, 1 Aug 2016 10:23:38 -0700 Subject: [PATCH] Updating config transformer for Admin API v1. --- convert_yaml.py | 28 +- yaml_conversion/converters.py | 70 ++++ .../lib/google/appengine/api/appinfo.py | 347 ++++++++++-------- .../google/appengine/api/appinfo_errors.py | 4 + .../lib/google/appengine/api/pagespeedinfo.py | 112 ------ yaml_conversion/yaml_schema_v1.py | 130 +++++++ .../{yaml_schema.py => yaml_schema_v1beta.py} | 24 +- 7 files changed, 442 insertions(+), 273 deletions(-) delete mode 100644 yaml_conversion/lib/google/appengine/api/pagespeedinfo.py create mode 100644 yaml_conversion/yaml_schema_v1.py rename yaml_conversion/{yaml_schema.py => yaml_schema_v1beta.py} (83%) diff --git a/convert_yaml.py b/convert_yaml.py index 8cf8d99..6851e23 100755 --- a/convert_yaml.py +++ b/convert_yaml.py @@ -19,24 +19,38 @@ convert_yaml.py app.yaml > app.json """ +import argparse import json -import os import sys import yaml -from yaml_conversion import yaml_schema +from yaml_conversion import yaml_schema_v1 +from yaml_conversion import yaml_schema_v1beta + + +API_VERSION_SCHEMAS = { + 'v1beta4': yaml_schema_v1beta, + 'v1beta5': yaml_schema_v1beta, + 'v1': yaml_schema_v1 +} def main(): - if len(sys.argv) != 2: - sys.stderr.write( - 'Usage: {0} \n'.format(os.path.basename(sys.argv[0]))) - sys.exit(1) + parser = argparse.ArgumentParser(description='Convert between legacy YAML ' + 'and public JSON representations of App ' + 'Engine versions') + parser.add_argument('input_file') + parser.add_argument('--api_version', dest='api_version', default='v1beta5', + choices=sorted(API_VERSION_SCHEMAS.keys())) - with open(sys.argv[1]) as input_file: + args = parser.parse_args() + + with open(args.input_file) as input_file: input_yaml = yaml.safe_load(input_file) + yaml_schema = API_VERSION_SCHEMAS[args.api_version] + converted_yaml = yaml_schema.SCHEMA.ConvertValue(input_yaml) json.dump(converted_yaml, sys.stdout, indent=2, sort_keys=True) diff --git a/yaml_conversion/converters.py b/yaml_conversion/converters.py index 128fb01..c389c18 100644 --- a/yaml_conversion/converters.py +++ b/yaml_conversion/converters.py @@ -49,6 +49,34 @@ 'apiEndpoint': _SCRIPT_FIELDS, } +_REQUEST_UTILIZATION_SCALING_FIELDS = ( + 'targetRequestCountPerSec', + 'targetConcurrentRequests', + 'targetRequestCountPerSecond', +) + +_DISK_UTILIZATION_SCALING_FIELDS = ( + 'targetWriteBytesPerSec', + 'targetWriteOpsPerSec', + 'targetReadBytesPerSec', + 'targetReadOpsPerSec', + 'targetWriteBytesPerSecond', + 'targetWriteOpsPerSecond', + 'targetReadBytesPerSecond', + 'targetReadOpsPerSecond', +) + +_NETWORK_UTILIZATION_SCALING_FIELDS = ( + 'targetSentBytesPerSec', + 'targetSentPacketsPerSec', + 'targetReceivedBytesPerSec', + 'targetReceivedPacketsPerSec', + 'targetSentBytesPerSecond', + 'targetSentPacketsPerSecond', + 'targetReceivedBytesPerSecond', + 'targetReceivedPacketsPerSecond', +) + def EnumConverter(prefix): """Create conversion function which translates from string to enum value. @@ -186,6 +214,48 @@ def ExpirationToDuration(value): return '%ss' % delta +def ConvertAutomaticScaling(automatic_scaling): + """Moves several VM-specific automatic scaling parameters to submessages. + + For example: + Input { + "targetSentPacketsPerSec": 10, + "targetReadOpsPerSec": 2, + "targetRequestCountPerSec": 3 + } + Output { + "networkUtilization": { + "targetSentPacketsPerSec": 10 + }, + "diskUtilization": { + "targetReadOpsPerSec": 2 + }, + "requestUtilization": { + "targetRequestCountPerSec": 3 + } + } + + Args: + automatic_scaling: Result of converting automatic_scaling according to + schema. + Returns: + AutomaticScaling which has moved network/disk utilization related fields to + submessage. + """ + def MoveFieldsTo(field_names, target_field_name): + target = {} + for field_name in field_names: + if field_name in automatic_scaling: + target[field_name] = automatic_scaling[field_name] + del automatic_scaling[field_name] + if target: + automatic_scaling[target_field_name] = target + MoveFieldsTo(_REQUEST_UTILIZATION_SCALING_FIELDS, 'requestUtilization') + MoveFieldsTo(_DISK_UTILIZATION_SCALING_FIELDS, 'diskUtilization') + MoveFieldsTo(_NETWORK_UTILIZATION_SCALING_FIELDS, 'networkUtilization') + return automatic_scaling + + def ConvertUrlHandler(handler): """Rejiggers the structure of the url handler based on its type. diff --git a/yaml_conversion/lib/google/appengine/api/appinfo.py b/yaml_conversion/lib/google/appengine/api/appinfo.py index 487b595..9d92382 100644 --- a/yaml_conversion/lib/google/appengine/api/appinfo.py +++ b/yaml_conversion/lib/google/appengine/api/appinfo.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -#!/usr/bin/python2.4 -# # Copyright 2007 Google Inc. All Rights Reserved. """AppInfo tools. @@ -43,15 +41,14 @@ import sys import wsgiref.util +# pylint: disable=g-import-not-at-top if os.environ.get('APPENGINE_RUNTIME') == 'python27': - from google.appengine.api import pagespeedinfo from google.appengine.api import validation from google.appengine.api import yaml_builder from google.appengine.api import yaml_listener from google.appengine.api import yaml_object else: # This case covers both Python 2.5 and unittests, which are 2.5 only. - from yaml_conversion.lib.google.appengine.api import pagespeedinfo from yaml_conversion.lib.google.appengine.api import validation from yaml_conversion.lib.google.appengine.api import yaml_builder from yaml_conversion.lib.google.appengine.api import yaml_listener @@ -60,6 +57,8 @@ from yaml_conversion.lib.google.appengine.api import appinfo_errors from yaml_conversion.lib.google.appengine.api import backendinfo +# pylint: enable=g-import-not-at-top + # Regular expression for matching url, file, url root regular expressions. # url_root is identical to url except it additionally imposes not ending with *. # TODO(user): url_root should generally allow a url but not a regex or glob. @@ -164,6 +163,7 @@ RUNTIME_RE_STRING = r'[a-z][a-z0-9\-]{0,29}' API_VERSION_RE_STRING = r'[\w.]{1,32}' +ENV_RE_STRING = r'[\w.]{1,32}' SOURCE_LANGUAGE_RE_STRING = r'[\w.\-]{1,32}' @@ -185,6 +185,7 @@ SECURE_HTTP = 'never' SECURE_HTTPS = 'always' SECURE_HTTP_OR_HTTPS = 'optional' +# Used for missing values; see http://b/issue?id=2073962. SECURE_DEFAULT = 'default' REQUIRE_MATCHING_FILE = 'require_matching_file' @@ -224,6 +225,7 @@ APPLICATION = 'application' PROJECT = 'project' # An alias for 'application' MODULE = 'module' +SERVICE = 'service' AUTOMATIC_SCALING = 'automatic_scaling' MANUAL_SCALING = 'manual_scaling' BASIC_SCALING = 'basic_scaling' @@ -239,6 +241,9 @@ MINOR_VERSION = 'minor_version' RUNTIME = 'runtime' API_VERSION = 'api_version' +ENV = 'env' +ENTRYPOINT = 'entrypoint' +RUNTIME_CONFIG = 'runtime_config' SOURCE_LANGUAGE = 'source_language' BUILTINS = 'builtins' INCLUDES = 'includes' @@ -259,7 +264,6 @@ API_CONFIG = 'api_config' CODE_LOCK = 'code_lock' ENV_VARIABLES = 'env_variables' -PAGESPEED = 'pagespeed' SOURCE_REPO_RE_STRING = r'^[a-z][a-z0-9\-\+\.]*:[^#]*$' SOURCE_REVISION_RE_STRING = r'^[0-9a-fA-F]+$' @@ -268,21 +272,38 @@ SOURCE_REFERENCES_MAX_SIZE = 2048 INSTANCE_CLASS = 'instance_class' -# Attributes for AutomaticScaling + +# Attributes for Standard App Engine (only) AutomaticScaling. MINIMUM_PENDING_LATENCY = 'min_pending_latency' MAXIMUM_PENDING_LATENCY = 'max_pending_latency' MINIMUM_IDLE_INSTANCES = 'min_idle_instances' MAXIMUM_IDLE_INSTANCES = 'max_idle_instances' MAXIMUM_CONCURRENT_REQUEST = 'max_concurrent_requests' -# Attributes for VM-based AutomaticScaling. -# See AutoscalingConfig in +# Attributes for Managed VMs (only) AutomaticScaling. These are very +# different than Standard App Engine because scaling settings are +# mapped to Cloud Autoscaler (as opposed to the clone scheduler). See +# AutoscalingConfig in MIN_NUM_INSTANCES = 'min_num_instances' MAX_NUM_INSTANCES = 'max_num_instances' COOL_DOWN_PERIOD_SEC = 'cool_down_period_sec' CPU_UTILIZATION = 'cpu_utilization' CPU_UTILIZATION_UTILIZATION = 'target_utilization' CPU_UTILIZATION_AGGREGATION_WINDOW_LENGTH_SEC = 'aggregation_window_length_sec' +# Managed VMs Richer Autoscaling. These (MVMs only) scaling settings +# are supported for both vm:true and env:2|flex, but are not yet +# publicly documented. +TARGET_NETWORK_SENT_BYTES_PER_SEC = 'target_network_sent_bytes_per_sec' +TARGET_NETWORK_SENT_PACKETS_PER_SEC = 'target_network_sent_packets_per_sec' +TARGET_NETWORK_RECEIVED_BYTES_PER_SEC = 'target_network_received_bytes_per_sec' +TARGET_NETWORK_RECEIVED_PACKETS_PER_SEC = ( + 'target_network_received_packets_per_sec') +TARGET_DISK_WRITE_BYTES_PER_SEC = 'target_disk_write_bytes_per_sec' +TARGET_DISK_WRITE_OPS_PER_SEC = 'target_disk_write_ops_per_sec' +TARGET_DISK_READ_BYTES_PER_SEC = 'target_disk_read_bytes_per_sec' +TARGET_DISK_READ_OPS_PER_SEC = 'target_disk_read_ops_per_sec' +TARGET_REQUEST_COUNT_PER_SEC = 'target_request_count_per_sec' +TARGET_CONCURRENT_REQUESTS = 'target_concurrent_requests' # Attributes for ManualScaling @@ -382,9 +403,8 @@ def non_deprecated_versions(self): 'django', 'http://www.djangoproject.com/', 'A full-featured web application framework for Python.', - ['1.2', '1.3', '1.4', '1.5'], + ['1.2', '1.3', '1.4', '1.5', '1.9'], latest_version='1.4', - experimental_versions=['1.5'], ), _VersionedLibrary( 'endpoints', @@ -412,7 +432,7 @@ def non_deprecated_versions(self): 'markupsafe', 'http://pypi.python.org/pypi/MarkupSafe', 'A XML/HTML/XHTML markup safe string for Python.', - ['0.15'], + ['0.15', '0.23'], latest_version='0.15', ), _VersionedLibrary( @@ -426,9 +446,9 @@ def non_deprecated_versions(self): 'MySQLdb', 'http://mysql-python.sourceforge.net/', 'A Python DB API v2.0 compatible interface to MySQL.', - ['1.2.4b4', '1.2.4'], - latest_version='1.2.4b4', - experimental_versions=['1.2.4b4', '1.2.4'] + ['1.2.4b4', '1.2.4', '1.2.5'], + latest_version='1.2.5', + experimental_versions=['1.2.4b4', '1.2.4', '1.2.5'] ), _VersionedLibrary( 'numpy', @@ -453,6 +473,14 @@ def non_deprecated_versions(self): latest_version='1.0', default_version='1.0', ), + _VersionedLibrary( + 'pytz', + 'https://pypi.python.org/pypi/pytz?', + 'A library for cross-platform timezone calculations', + ['2016.4'], + latest_version='2016.4', + default_version='2016.4', + ), _VersionedLibrary( 'crcmod', 'http://crcmod.sourceforge.net/', @@ -473,7 +501,7 @@ def non_deprecated_versions(self): 'pycrypto', 'https://www.dlitz.net/software/pycrypto/', 'A library of cryptography functions such as random number generation.', - ['2.3', '2.6'], + ['2.3', '2.6', '2.6.1'], latest_version='2.6', ), _VersionedLibrary( @@ -487,8 +515,9 @@ def non_deprecated_versions(self): 'ssl', 'http://docs.python.org/dev/library/ssl.html', 'The SSL socket wrapper built-in module.', - ['2.7'], + ['2.7', '2.7.11'], latest_version='2.7', + default_version='2.7.11', ), _VersionedLibrary( 'webapp2', @@ -529,8 +558,7 @@ def non_deprecated_versions(self): ('matplotlib', 'latest'): [('numpy', 'latest')], } -_USE_VERSION_FORMAT = ('use one of: "%s" or "latest" ' - '("latest" recommended for development only)') +_USE_VERSION_FORMAT = ('use one of: "%s"') # See RFC 2616 section 2.2. @@ -568,14 +596,13 @@ def non_deprecated_versions(self): # trailing NULL character, which is why this is not 2048. _MAX_URL_LENGTH = 2047 -# The following suite of functions provide information about the sets of -# supported runtimes. These sets can be modified by external systems, -# notably gcloud. +# We allow certain headers to be larger than the normal limit of 8192 bytes. +_MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS = 10240 _CANNED_RUNTIMES = ('contrib-dart', 'dart', 'go', 'php', 'php55', 'python', - 'python27', 'java', 'java7', 'vm', 'custom', 'nodejs') + 'python27', 'python-compat', 'java', 'java7', 'vm', + 'custom', 'nodejs', 'ruby') _all_runtimes = _CANNED_RUNTIMES -_vm_runtimes = _CANNED_RUNTIMES def GetAllRuntimes(): @@ -589,35 +616,6 @@ def GetAllRuntimes(): return _all_runtimes -def SetAllRuntimes(runtimes): - """Sets the list of all valid runtimes. - - Args: - runtimes: Tuple of strings defining the names of all valid runtimes. - """ - global _all_runtimes - _all_runtimes = runtimes - - -def GetVmRuntimes(): - """Returns the list of runtimes for the vm_runtimes field. - - Returns: - Tuple of strings. - """ - return _vm_runtimes - - -def SetVmRuntimes(runtimes): - """Sets the list of all runtimes valid for the vm_runtimes field. - - Args: - runtimes: Tuple of strings defining all valid vm runtimes. - """ - global _vm_runtimes - _vm_runtimes = runtimes - - class HandlerBase(validation.Validated): """Base class for URLMap and ApiConfigHandler.""" ATTRIBUTES = { @@ -669,6 +667,11 @@ class HttpHeadersDict(validation.ValidatedDict): MAX_HEADER_LENGTH = 500 MAX_HEADER_VALUE_LENGTHS = { + 'content-security-policy': _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, + 'x-content-security-policy': _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, + 'x-webkit-csp': _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, + 'content-security-policy-report-only': + _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, 'set-cookie': _MAX_COOKIE_LENGTH, 'set-cookie2': _MAX_COOKIE_LENGTH, 'location': _MAX_URL_LENGTH} @@ -1087,6 +1090,13 @@ def AssertUniqueContentType(self): ' also specified a mime_type of %r.' % (content_type, self.mime_type)) def FixSecureDefaults(self): + """Force omitted 'secure: ...' handler fields to 'secure: optional'. + + The effect is that handler.secure is never equal to the (nominal) + default. + + See http://b/issue?id=2073962. + """ if self.secure == SECURE_DEFAULT: self.secure = SECURE_HTTP_OR_HTTPS @@ -1357,21 +1367,22 @@ def CheckInitialized(self): if self.name not in _NAME_TO_SUPPORTED_LIBRARY: raise appinfo_errors.InvalidLibraryName( 'the library "%s" is not supported' % self.name) - supported_library = _NAME_TO_SUPPORTED_LIBRARY[self.name] - if self.version != 'latest': - if self.version not in supported_library.supported_versions: - raise appinfo_errors.InvalidLibraryVersion( - ('%s version "%s" is not supported, ' + _USE_VERSION_FORMAT) % ( - self.name, - self.version, - '", "'.join(supported_library.non_deprecated_versions))) - elif self.version in supported_library.deprecated_versions: - logging.warning( - ('%s version "%s" is deprecated, ' + _USE_VERSION_FORMAT) % ( - self.name, - self.version, - '", "'.join(supported_library.non_deprecated_versions))) + if self.version == 'latest': + self.version = supported_library.latest_version + elif self.version not in supported_library.supported_versions: + raise appinfo_errors.InvalidLibraryVersion( + ('%s version "%s" is not supported, ' + _USE_VERSION_FORMAT) % ( + self.name, + self.version, + '", "'.join(supported_library.non_deprecated_versions))) + elif self.version in supported_library.deprecated_versions: + use_vers = '", "'.join(supported_library.non_deprecated_versions) + logging.warning( + '%s version "%s" is deprecated, ' + _USE_VERSION_FORMAT, + self.name, + self.version, + use_vers) class CpuUtilization(validation.Validated): @@ -1400,6 +1411,26 @@ class AutomaticScaling(validation.Validated): COOL_DOWN_PERIOD_SEC: validation.Optional( validation.Range(60, sys.maxint, int)), CPU_UTILIZATION: validation.Optional(CpuUtilization), + TARGET_NETWORK_SENT_BYTES_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_NETWORK_SENT_PACKETS_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_NETWORK_RECEIVED_BYTES_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_NETWORK_RECEIVED_PACKETS_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_DISK_WRITE_BYTES_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_DISK_WRITE_OPS_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_DISK_READ_BYTES_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_DISK_READ_OPS_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_REQUEST_COUNT_PER_SEC: + validation.Optional(validation.Range(1, sys.maxint)), + TARGET_CONCURRENT_REQUESTS: + validation.Optional(validation.Range(1, sys.maxint)), } @@ -1418,7 +1449,24 @@ class BasicScaling(validation.Validated): } +class RuntimeConfig(validation.ValidatedDict): + """Class for "vanilla" runtime configuration. + + Fields used vary by runtime, so we delegate validation to the per-runtime + build processes. + + These are intended to be used during Dockerfile generation, not after VM boot. + """ + + KEY_VALIDATOR = validation.Regex('[a-zA-Z_][a-zA-Z0-9_]*') + VALUE_VALIDATOR = str + + class VmSettings(validation.ValidatedDict): + """Class for VM settings. + + We don't validate these further here. They're validated server side. + """ KEY_VALIDATOR = validation.Regex('[a-zA-Z_][a-zA-Z0-9_]*') VALUE_VALIDATOR = str @@ -1435,6 +1483,13 @@ def Merge(cls, vm_settings_one, vm_settings_two): class BetaSettings(VmSettings): + """Class for Beta (internal or unreleased) settings. + + This class is meant to replace VmSettings eventually. + All new beta settings must be registered in shared_constants.py. + + We don't validate these further here. They're validated server side. + """ @classmethod def Merge(cls, beta_settings_one, beta_settings_two): @@ -1470,78 +1525,6 @@ def Merge(cls, env_variables_one, env_variables_two): if result_env_variables else None) -def VmSafeSetRuntime(appyaml, runtime): - """Sets the runtime while respecting vm runtimes rules for runtime settings. - - Args: - appyaml: AppInfoExternal instance, which will be modified. - runtime: The runtime to use. - - Returns: - The passed in appyaml (which has been modified). - """ - if appyaml.vm: - if not appyaml.vm_settings: - appyaml.vm_settings = VmSettings() - - # Both 'dart' and 'contrib-dart' runtimes are known as 'dart', and - # always use a Docker image. - if runtime == 'dart' or runtime == 'contrib-dart': - runtime = 'dart' - appyaml.vm_settings['has_docker_image'] = True - # Convert all other runtimes that are not legal for the vm_runtimes field - # to "custom". - elif runtime not in GetVmRuntimes(): - runtime = 'custom' - - # Patch up vm runtime setting. Copy 'runtime' to 'vm_runtime' and - # set runtime to the string 'vm'. - appyaml.vm_settings['vm_runtime'] = runtime - appyaml.runtime = 'vm' - else: - appyaml.runtime = runtime - return appyaml - - -def NormalizeVmSettings(appyaml): - """Normalize Vm settings. - - Args: - appyaml: AppInfoExternal instance. - - Returns: - Normalized app yaml. - """ - # NOTE(user): In the input files, 'vm' is not a type of runtime, but - # rather is specified as "vm: true|false". In the code, 'vm' - # is represented as a value of AppInfoExternal.runtime. - # NOTE(user): This hack is only being applied after the parsing of - # AppInfoExternal. If the 'vm' attribute can ever be specified in the - # AppInclude, then this processing will need to be done there too. - if appyaml.vm: - if not appyaml.vm_settings: - appyaml.vm_settings = VmSettings() - - if 'vm_runtime' not in appyaml.vm_settings: - appyaml = VmSafeSetRuntime(appyaml, appyaml.runtime) - - # Copy fields that are automatically added by the SDK or this class - # to beta_settings. - if hasattr(appyaml, 'beta_settings') and appyaml.beta_settings: - # Only copy if beta_settings already exists, because we have logic in - # appversion.py to discard all of vm_settings if anything is in - # beta_settings. So we won't create an empty one just to add these - # fields. - for field in ['vm_runtime', - 'has_docker_image', - 'image', - 'module_yaml_path']: - if field not in appyaml.beta_settings and field in appyaml.vm_settings: - appyaml.beta_settings[field] = appyaml.vm_settings[field] - - return appyaml - - def ValidateSourceReference(ref): """Determines if a source reference is valid. @@ -1749,12 +1732,14 @@ def MergeAppYamlAppInclude(cls, appyaml, appinclude): appyaml.handlers.extend(tail) appyaml = cls._CommonMergeOps(appyaml, appinclude) - return NormalizeVmSettings(appyaml) + appyaml.NormalizeVmSettings() + return appyaml @classmethod def MergeAppIncludes(cls, appinclude_one, appinclude_two): - """This function merges the non-referential state of the provided AppInclude - objects. That is, builtins and includes directives are not preserved, but + """Merges the non-referential state of the provided AppInclude. + + That is, builtins and includes directives are not preserved, but any static objects are copied into an aggregate AppInclude object that preserves the directives of both provided AppInclude objects. @@ -1832,11 +1817,18 @@ class AppInfoExternal(validation.Validated): # An alias for APPLICATION. PROJECT: validation.Optional(APPLICATION_RE_STRING), MODULE: validation.Optional(MODULE_ID_RE_STRING), + # 'service' will replace 'module' soon + SERVICE: validation.Optional(MODULE_ID_RE_STRING), VERSION: validation.Optional(MODULE_VERSION_ID_RE_STRING), RUNTIME: validation.Optional(RUNTIME_RE_STRING), # A new api_version requires a release of the dev_appserver, so it # is ok to hardcode the version names here. - API_VERSION: API_VERSION_RE_STRING, + API_VERSION: validation.Optional(API_VERSION_RE_STRING), + # The App Engine environment to run this version in. (VM vs. non-VM, etc.) + ENV: validation.Optional(ENV_RE_STRING), + # The SDK will use this for generated Dockerfiles + ENTRYPOINT: validation.Optional(validation.Type(str)), + RUNTIME_CONFIG: validation.Optional(RuntimeConfig), INSTANCE_CLASS: validation.Optional(_INSTANCE_CLASS_REGEX), SOURCE_LANGUAGE: validation.Optional( validation.Regex(SOURCE_LANGUAGE_RE_STRING)), @@ -1873,10 +1865,8 @@ class AppInfoExternal(validation.Validated): API_CONFIG: validation.Optional(ApiConfigHandler), CODE_LOCK: validation.Optional(bool), ENV_VARIABLES: validation.Optional(EnvironmentVariables), - PAGESPEED: validation.Optional(pagespeedinfo.PagespeedEntry), } - def CheckInitialized(self): """Performs non-regex-based validation. @@ -1889,6 +1879,7 @@ def CheckInitialized(self): can be used. - That the version name doesn't start with BUILTIN_NAME_PREFIX - If redirect_http_response_code exists, it is in the list of valid 300s. + - That module and service aren't both set Raises: DuplicateLibrary: if the name library name is specified more than once. @@ -1902,9 +1893,10 @@ def CheckInitialized(self): present. RuntimeDoesNotSupportLibraries: if libraries clause is used for a runtime that does not support it (e.g. python25). + ModuleAndServiceDefined: if both 'module' and 'service' keywords are used. """ super(AppInfoExternal, self).CheckInitialized() - if self.runtime is None and not self.vm: + if self.runtime is None and not self.IsVm(): raise appinfo_errors.MissingRuntimeError( 'You must specify a "runtime" field for non-vm applications.') elif self.runtime is None: @@ -1912,13 +1904,16 @@ def CheckInitialized(self): # we know that it's been defaulted) self.runtime = 'custom' if (not self.handlers and not self.builtins and not self.includes - and not self.vm): + and not self.IsVm()): raise appinfo_errors.MissingURLMapping( 'No URLMap entries found in application configuration') if self.handlers and len(self.handlers) > MAX_URL_MAPS: raise appinfo_errors.TooManyURLMappings( 'Found more than %d URLMap entries in application configuration' % MAX_URL_MAPS) + if self.service and self.module: + raise appinfo_errors.ModuleAndServiceDefined( + 'Cannot define both "module" and "service" in configuration') vm_runtime_python27 = ( self.runtime == 'vm' and @@ -1932,7 +1927,7 @@ def CheckInitialized(self): if (self.threadsafe is None and (self.runtime == 'python27' or vm_runtime_python27)): raise appinfo_errors.MissingThreadsafe( - 'threadsafe must be present and set to either "yes" or "no"') + 'threadsafe must be present and set to a true or false YAML value') if self.auto_id_policy == DATASTORE_ID_POLICY_LEGACY: datastore_auto_ids_url = ('http://developers.google.com/' @@ -2034,10 +2029,6 @@ def GetNormalizedLibraries(self): if library.default_version and library.name not in enabled_libraries: libraries.append(Library(name=library.name, version=library.default_version)) - for library in libraries: - if library.version == 'latest': - library.version = _NAME_TO_SUPPORTED_LIBRARY[ - library.name].supported_versions[-1] return libraries def ApplyBackendSettings(self, backend_name): @@ -2095,6 +2086,57 @@ def GetEffectiveRuntime(self): return self.beta_settings.get('vm_runtime') return self.runtime + def SetEffectiveRuntime(self, runtime): + """Sets the runtime while respecting vm runtimes rules for runtime settings. + + Args: + runtime: The runtime to use. + """ + if self.IsVm(): + if not self.vm_settings: + self.vm_settings = VmSettings() + + # Patch up vm runtime setting. Copy 'runtime' to 'vm_runtime' and + # set runtime to the string 'vm'. + self.vm_settings['vm_runtime'] = runtime + self.runtime = 'vm' + else: + self.runtime = runtime + + def NormalizeVmSettings(self): + """Normalize Vm settings. + """ + # NOTE(user): In the input files, 'vm' is not a type of runtime, but + # rather is specified as "vm: true|false". In the code, 'vm' + # is represented as a value of AppInfoExternal.runtime. + # NOTE(user): This hack is only being applied after the parsing of + # AppInfoExternal. If the 'vm' attribute can ever be specified in the + # AppInclude, then this processing will need to be done there too. + if self.IsVm(): + if not self.vm_settings: + self.vm_settings = VmSettings() + + if 'vm_runtime' not in self.vm_settings: + self.SetEffectiveRuntime(self.runtime) + + # Copy fields that are automatically added by the SDK or this class + # to beta_settings. + if hasattr(self, 'beta_settings') and self.beta_settings: + # Only copy if beta_settings already exists, because we have logic in + # appversion.py to discard all of vm_settings if anything is in + # beta_settings. So we won't create an empty one just to add these + # fields. + for field in ['vm_runtime', + 'has_docker_image', + 'image', + 'module_yaml_path']: + if field not in self.beta_settings and field in self.vm_settings: + self.beta_settings[field] = self.vm_settings[field] + + # TODO(user): env replaces vm. Remove vm when field is removed. + def IsVm(self): + return (self.vm or + self.env in ['2', 'flex', 'flexible']) def ValidateHandlers(handlers, is_include_file=False): """Validates a list of handler (URLMap) objects. @@ -2165,7 +2207,8 @@ def LoadSingleAppInfo(app_info): appyaml.application = appyaml.project appyaml.project = None - return NormalizeVmSettings(appyaml) + appyaml.NormalizeVmSettings() + return appyaml class AppInfoSummary(validation.Validated): diff --git a/yaml_conversion/lib/google/appengine/api/appinfo_errors.py b/yaml_conversion/lib/google/appengine/api/appinfo_errors.py index a7fb5c5..13af99c 100644 --- a/yaml_conversion/lib/google/appengine/api/appinfo_errors.py +++ b/yaml_conversion/lib/google/appengine/api/appinfo_errors.py @@ -37,6 +37,10 @@ class EmptyConfigurationFile(Error): """Tried to load empty configuration file""" +class ModuleAndServiceDefined(Error): + """Configuration has both 'module' and 'service' instead of just one.""" + + class MultipleConfigurationFile(Error): """Tried to load configuration file with multiple AppInfo objects""" diff --git a/yaml_conversion/lib/google/appengine/api/pagespeedinfo.py b/yaml_conversion/lib/google/appengine/api/pagespeedinfo.py deleted file mode 100644 index 3e8a9bb..0000000 --- a/yaml_conversion/lib/google/appengine/api/pagespeedinfo.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2015 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS-IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/usr/bin/python2.4 -# -# Copyright 2012 Google Inc. All Rights Reserved. - -"""PageSpeed configuration tools. - -Library for parsing pagespeed configuration data from app.yaml and working -with these in memory. -""" - - - -# WARNING: This file is externally viewable by our users. All comments from -# this file will be stripped. The docstrings will NOT. Do not put sensitive -# information in docstrings. If you must communicate internal information in -# this source file, please place them in comments only. - - -from yaml_conversion.lib.google.appengine.api import validation -from yaml_conversion.lib.google.appengine.api import yaml_builder -from yaml_conversion.lib.google.appengine.api import yaml_listener -from yaml_conversion.lib.google.appengine.api import yaml_object - -_URL_BLACKLIST_REGEX = r'http(s)?://\S{0,499}' -_REWRITER_NAME_REGEX = r'[a-zA-Z0-9_]+' -_DOMAINS_TO_REWRITE_REGEX = r'(http(s)?://)?[-a-zA-Z0-9_.*]+(:\d+)?' - -URL_BLACKLIST = 'url_blacklist' -ENABLED_REWRITERS = 'enabled_rewriters' -DISABLED_REWRITERS = 'disabled_rewriters' -DOMAINS_TO_REWRITE = 'domains_to_rewrite' - - -class MalformedPagespeedConfiguration(Exception): - """Configuration file for PageSpeed API is malformed.""" - - -# Note: we don't validate the names of enabled/disabled rewriters in this code, -# since the list is subject to change (namely, we add new rewriters, and want to -# be able to make them available to users without tying ourselves to App -# Engine's release cycle, or worry about keeping the two lists in sync). -class PagespeedEntry(validation.Validated): - """Describes the format of a pagespeed configuration from a yaml file. - - URL blacklist entries are patterns (with '?' and '*' as wildcards). Any URLs - that match a pattern on the blacklist will not be optimized by PageSpeed. - - Rewriter names are strings (like 'CombineCss' or 'RemoveComments') describing - individual PageSpeed rewriters. A full list of valid rewriter names can be - found in the PageSpeed documentation. - - The domains-to-rewrite list is a whitelist of domain name patterns with '*' as - a wildcard, optionally starting with 'http://' or 'https://'. If no protocol - is given, 'http://' is assumed. A resource will only be rewritten if it is on - the same domain as the HTML that references it, or if its domain is on the - domains-to-rewrite list. - """ - ATTRIBUTES = { - URL_BLACKLIST: validation.Optional( - validation.Repeated(validation.Regex(_URL_BLACKLIST_REGEX))), - ENABLED_REWRITERS: validation.Optional( - validation.Repeated(validation.Regex(_REWRITER_NAME_REGEX))), - DISABLED_REWRITERS: validation.Optional( - validation.Repeated(validation.Regex(_REWRITER_NAME_REGEX))), - DOMAINS_TO_REWRITE: validation.Optional( - validation.Repeated(validation.Regex(_DOMAINS_TO_REWRITE_REGEX))), - } - - -def LoadPagespeedEntry(pagespeed_entry, open_fn=None): - """Load a yaml file or string and return a PagespeedEntry. - - Args: - pagespeed_entry: The contents of a pagespeed entry from a yaml file - as a string, or an open file object. - open_fn: Function for opening files. Unused. - - Returns: - A PagespeedEntry instance which represents the contents of the parsed yaml. - - Raises: - yaml_errors.EventError: An error occured while parsing the yaml. - MalformedPagespeedConfiguration: The configuration is parseable but invalid. - """ - builder = yaml_object.ObjectBuilder(PagespeedEntry) - handler = yaml_builder.BuilderHandler(builder) - listener = yaml_listener.EventListener(handler) - listener.Parse(pagespeed_entry) - - parsed_yaml = handler.GetResults() - if not parsed_yaml: - return PagespeedEntry() - - if len(parsed_yaml) > 1: - raise MalformedPagespeedConfiguration( - 'Multiple configuration sections in the yaml') - - return parsed_yaml[0] diff --git a/yaml_conversion/yaml_schema_v1.py b/yaml_conversion/yaml_schema_v1.py new file mode 100644 index 0000000..5adf7c0 --- /dev/null +++ b/yaml_conversion/yaml_schema_v1.py @@ -0,0 +1,130 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Definition for conversion between legacy YAML and the API JSON formats.""" + +from yaml_conversion import converters as c +from yaml_conversion import schema as s + + +SCHEMA = s.Message( + api_config=s.Message( + url=s.Value(converter=c.ToJsonString), + login=s.Value(converter=c.EnumConverter('LOGIN')), + secure=s.Value('security_level', converter=c.EnumConverter('SECURE')), + auth_fail_action=s.Value(converter=c.EnumConverter('AUTH_FAIL_ACTION')), + script=s.Value(converter=c.ToJsonString)), + auto_id_policy=s.Value('beta_settings', + lambda val: {'auto_id_policy': val}), + automatic_scaling=s.Message( + converter=c.ConvertAutomaticScaling, + cool_down_period_sec=s.Value('cool_down_period', + converter=c.SecondsToDuration), + cpu_utilization=s.Message( + target_utilization=s.Value(), + aggregation_window_length_sec=s.Value('aggregation_window_length', + converter=c.SecondsToDuration) + ), + max_num_instances=s.Value('max_total_instances'), + min_pending_latency=s.Value(converter=c.LatencyToDuration), + min_idle_instances=s.Value(converter= + c.StringToInt(handle_automatic=True)), + max_idle_instances=s.Value(converter= + c.StringToInt(handle_automatic=True)), + max_pending_latency=s.Value(converter=c.LatencyToDuration), + max_concurrent_requests=s.Value(converter=c.StringToInt()), + min_num_instances=s.Value('min_total_instances'), + target_network_sent_bytes_per_sec=s.Value( + 'target_sent_bytes_per_second'), + target_network_sent_packets_per_sec=s.Value( + 'target_sent_packets_per_second'), + target_network_received_bytes_per_sec=s.Value( + 'target_received_bytes_per_second'), + target_network_received_packets_per_sec=s.Value( + 'target_received_packets_per_second'), + target_disk_write_bytes_per_sec=s.Value( + 'target_write_bytes_per_second'), + target_disk_write_ops_per_sec=s.Value( + 'target_write_ops_per_second'), + target_disk_read_bytes_per_sec=s.Value( + 'target_read_bytes_per_second'), + target_disk_read_ops_per_sec=s.Value( + 'target_read_ops_per_second'), + target_request_count_per_sec=s.Value( + 'target_request_count_per_second'), + target_concurrent_requests=s.Value()), + basic_scaling=s.Message( + idle_timeout=s.Value(converter=c.IdleTimeoutToDuration), + max_instances=s.Value(converter=c.StringToInt())), + beta_settings=s.Map(), + default_expiration=s.Value(converter=c.ExpirationToDuration), + env=s.Value(), + env_variables=s.Map(), + error_handlers=s.RepeatedField(element=s.Message( + error_code=s.Value(converter=c.EnumConverter('ERROR_CODE')), + file=s.Value('static_file', converter=c.ToJsonString), + mime_type=s.Value(converter=c.ToJsonString))), + # Restructure the handler after it's complete, since this is more + # complicated than a simple rename. + handlers=s.RepeatedField(element=s.Message( + converter=c.ConvertUrlHandler, + auth_fail_action=s.Value(converter=c.EnumConverter('AUTH_FAIL_ACTION')), + static_dir=s.Value(converter=c.ToJsonString), + secure=s.Value('security_level', converter=c.EnumConverter('SECURE')), + redirect_http_response_code=s.Value( + converter=c.EnumConverter('REDIRECT_HTTP_RESPONSE_CODE')), + http_headers=s.Map(), + url=s.Value('url_regex'), + expiration=s.Value(converter=c.ExpirationToDuration), + static_files=s.Value('path', converter=c.ToJsonString), + script=s.Value('script_path', converter=c.ToJsonString), + upload=s.Value('upload_path_regex', converter=c.ToJsonString), + api_endpoint=s.Value(), + application_readable=s.Value(), + position=s.Value(), + login=s.Value(converter=c.EnumConverter('LOGIN')), + mime_type=s.Value(converter=c.ToJsonString), + require_matching_file=s.Value())), + health_check=s.Message( + check_interval_sec=s.Value('check_interval', + converter=c.SecondsToDuration), + timeout_sec=s.Value('timeout', converter=c.SecondsToDuration), + healthy_threshold=s.Value(), + enable_health_check=s.Value('disable_health_check', converter=c.Not), + unhealthy_threshold=s.Value(), + host=s.Value(converter=c.ToJsonString), + restart_threshold=s.Value()), + inbound_services=s.RepeatedField(element=s.Value( + converter=c.EnumConverter('INBOUND_SERVICE'))), + instance_class=s.Value(converter=c.ToJsonString), + libraries=s.RepeatedField(element=s.Message( + version=s.Value(converter=c.ToJsonString), + name=s.Value(converter=c.ToJsonString))), + manual_scaling=s.Message( + instances=s.Value(converter=c.StringToInt())), + network=s.Message( + instance_tag=s.Value(converter=c.ToJsonString), + name=s.Value(converter=c.ToJsonString), + forwarded_ports=s.RepeatedField(element=s.Value(converter= + c.ToJsonString))), + nobuild_files=s.Value('nobuild_files_regex', converter=c.ToJsonString), + resources=s.Message( + memory_gb=s.Value(), + disk_size_gb=s.Value('disk_gb'), + cpu=s.Value()), + runtime=s.Value(converter=c.ToJsonString), + threadsafe=s.Value(), + version=s.Value('id', converter=c.ToJsonString), + vm=s.Value(), + vm_settings=s.Map('beta_settings')) diff --git a/yaml_conversion/yaml_schema.py b/yaml_conversion/yaml_schema_v1beta.py similarity index 83% rename from yaml_conversion/yaml_schema.py rename to yaml_conversion/yaml_schema_v1beta.py index 496aa3b..11abf5d 100644 --- a/yaml_conversion/yaml_schema.py +++ b/yaml_conversion/yaml_schema_v1beta.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Definition for conversion between legacy YAML and One Platform protos.""" +"""Definition for conversion between legacy YAML and the API JSON formats.""" from yaml_conversion import converters as c from yaml_conversion import schema as s @@ -28,6 +28,7 @@ auto_id_policy=s.Value('beta_settings', lambda val: {'auto_id_policy': val}), automatic_scaling=s.Message( + converter=c.ConvertAutomaticScaling, cool_down_period_sec=s.Value('cool_down_period', converter=c.SecondsToDuration), cpu_utilization=s.Message( @@ -43,12 +44,31 @@ c.StringToInt(handle_automatic=True)), max_pending_latency=s.Value(converter=c.LatencyToDuration), max_concurrent_requests=s.Value(converter=c.StringToInt()), - min_num_instances=s.Value('min_total_instances')), + min_num_instances=s.Value('min_total_instances'), + target_network_sent_bytes_per_sec=s.Value( + 'target_sent_bytes_per_sec'), + target_network_sent_packets_per_sec=s.Value( + 'target_sent_packets_per_sec'), + target_network_received_bytes_per_sec=s.Value( + 'target_received_bytes_per_sec'), + target_network_received_packets_per_sec=s.Value( + 'target_received_packets_per_sec'), + target_disk_write_bytes_per_sec=s.Value( + 'target_write_bytes_per_sec'), + target_disk_write_ops_per_sec=s.Value( + 'target_write_ops_per_sec'), + target_disk_read_bytes_per_sec=s.Value( + 'target_read_bytes_per_sec'), + target_disk_read_ops_per_sec=s.Value( + 'target_read_ops_per_sec'), + target_request_count_per_sec=s.Value(), + target_concurrent_requests=s.Value()), basic_scaling=s.Message( idle_timeout=s.Value(converter=c.IdleTimeoutToDuration), max_instances=s.Value(converter=c.StringToInt())), beta_settings=s.Map(), default_expiration=s.Value(converter=c.ExpirationToDuration), + env=s.Value(), env_variables=s.Map(), error_handlers=s.RepeatedField(element=s.Message( error_code=s.Value(converter=c.EnumConverter('ERROR_CODE')),