Browse files

some further updates, setup vendor packages

  • Loading branch information...
1 parent ef98a7f commit fb7abc6700872713e7f69cacc0f2478c94595f53 @ianb committed May 10, 2012
Showing with 229 additions and 30 deletions.
  1. +115 −22 apppkg/__init__.py
  2. +19 −8 apppkg/init.py
  3. +58 −0 apppkg/readme-layout.txt
  4. +37 −0 apppkg/scriptfixup.py
View
137 apppkg/__init__.py
@@ -70,29 +70,29 @@ def wsgi_application(self):
@property
def config_required(self):
"""Bool: is the configuration required"""
- return self.config.get('config', {}).get('required')
+ return self.description.get('config', {}).get('required')
@property
def config_template(self):
"""Path: where a configuration template exists"""
## FIXME: should this be a command?
- v = self.config.get('config', {}).get('template')
+ v = self.description.get('config', {}).get('template')
if v:
return self.abspath(v)
return None
@property
def config_validator(self):
"""Object: validator for the configuration"""
- v = self.config.get('config', {}).get('validator')
+ v = self.description.get('config', {}).get('validator')
if v:
return CommandReference(self, v, 'config.validator')
return None
@property
def config_default_dir(self):
"""Path: default configuration if no other is provided"""
- dir = self.config.get('config', {}).get('default')
+ dir = self.description.get('config', {}).get('default')
if dir:
return self.abspath(dir)
return None
@@ -108,7 +108,7 @@ def activate_path(self, venv_path=None):
if isinstance(dirs, basestring):
dirs = [dirs]
dirs = [self.abspath(dir) for dir in dirs]
- add_paths = list(self.add_paths)
+ add_paths = list(dirs)
add_paths.extend([
self.abspath('lib/python%s' % sys.version[:3]),
self.abspath('lib/python%s/site-packages' % sys.version[:3]),
@@ -182,7 +182,12 @@ def call_script(self, script_path, arguments, env_overrides=None, cwd=None, pyth
environ=env, cwd=cwd)
return proc
- ## FIXME: need something to run "commands" (as defined in the spec)
+ def initialize_for_script(self):
+ """Initializes the environment for the purposes of a script"""
+ venv = os.path.join(self.path, '.virtualenv')
+ if not os.path.exists(venv):
+ venv = None
+ self.activate_path(venv)
class Requires(object):
@@ -268,6 +273,10 @@ def install_pip(self, venv_path, make_venv=False):
class CommandReference(object):
+ """Represents a reference to a command or object in the
+ configuration. Can be executed with ``.run()`` or an object
+ extracted with ``.get_object()``
+ """
def __init__(self, app, ref, name):
self.app = app
@@ -284,15 +293,15 @@ def parse_ref_type(self, ref):
if ref.startswith('/') or ref.startswith('url:'):
if ref.startswith('url:'):
ref = ref[4:]
- return 'url', ref
+ return 'url', (ref, None)
if ref.startswith('script:'):
ref = ref.split(':', 1)
path = self.app.abspath(ref)
with open(path) as fp:
first = fp.readline()
if first.startswith('#!') and 'python' in first:
return 'py', (ref, None)
- return 'script', ref
+ return 'script', (ref, None)
if ref.endswith('.py') or '.py:' in ref or ref.startswith('pyscript:'):
if ref.startswith('pyscript:'):
ref = ref[len('pyscript:'):]
@@ -305,26 +314,43 @@ def parse_ref_type(self, ref):
if self._PY_MOD_RE.search(ref) or ref.startswith('py:'):
if ref.startswith('py:'):
ref = ref[3:]
+ if ':' in ref:
+ path, extra = ref.split(':', 1)
+ else:
+ path = ref
+ extra = None
return 'py', (path, extra)
- def run(self, *args):
+ def run(self, *args, **kw):
"""Runs the command, returning (text_output, extra_data), or
raising an exception"""
- return getattr(self, 'run_' + self.ref_type)(self.app.environment, *args)
+ return getattr(self, 'run_' + self.ref_type)(self.app.environment, *args, **kw)
- def run_url(self, *args):
+ def run_url(self, *args, **kw):
obj = self.app.wsgi_application.get_object()
- if '?' in self.ref:
- path, query_string = self.ref.split('?', 1)
+ url = self.ref_data[0]
+ if '?' in url:
+ path, query_string = url.split('?', 1)
else:
- path, query_string = self.ref, ''
+ path, query_string = url, ''
+ all_args = []
if args:
+ for a in args:
+ all_args.append(None, a)
+ if kw:
+ for name, value in sorted(kw.items()):
+ all_args.append(name, value)
+ if all_args:
body = []
- for item in args:
- if isinstance(item, (int, float, str, unicode)):
- body.append(urllib.quote(str(item)))
+ for name, value in args:
+ if isinstance(value, (int, float, str, unicode)):
+ value = urllib.quote(str(value))
+ else:
+ value = urllib.quote(json.dumps(value))
+ if name:
+ body.append('%s=%s' % (urllib.quote(name), value))
else:
- body.append(urllib.quote(json.dumps(item)))
+ body.append(value)
body = '&'.join(body)
else:
body = ''
@@ -334,7 +360,7 @@ def run_url(self, *args):
'wsgi.input': StringIO(body),
'SERVER_NAME': 'localhost',
'SERVER_PORT': '0',
- 'HTTP_HOST': 'http://localhost:0'
+ 'HTTP_HOST': 'http://localhost:0',
'SCRIPT_NAME': '',
'PATH_INFO': urllib.unquote(path),
'QUERY_STRING': query_string,
@@ -360,10 +386,76 @@ def start_response(status, headers, exc_info=None):
metadata = {'headers': headers, 'status': status}
return output, metadata
- def run_script(self, *args):
- cmd = [self.app.abspath(self.ref)] + list(args)
+ def run_script(self, *args, **kw):
+ if '.cmd' in kw:
+ cmd = [kw.pop('.cmd')]
+ else:
+ cmd = [self.app.abspath(self.ref)]
+ if '.exe' in kw:
+ cmd.insert(0, kw.pop('.exe'))
+ for name, value in sorted(kw.items()):
+ if len(name) == 1:
+ name = '-%s' % name
+ else:
+ name = '--%s' % name
+ cmd.append(name)
+ if value is True:
+ pass
+ elif isinstance(value, (int, float, str, unicode)):
+ cmd.append(str(value))
+ elif isinstance(value, basestring):
+ cmd.append(value)
+ else:
+ cmd.append(json.dumps(value))
+ cmd += list(args)
proc = subprocess.Popen(
-
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ cwd=self.app.path)
+ ## FIXME: should set env variables
+ stdout, stderr = proc.communicate()
+ metadata = {'stderr': stderr}
+ return stdout, metadata
+
+ def run_pyscript(self, *args, **kw):
+ env = self.app.environment
+ if env:
+ executable = env.base_python_exe
+ else:
+ executable = sys.executable
+ kw['.exe'] = executable
+ kw['.cmd'] = self.ref[0]
+ if self.ref[1] is not None:
+ raise Exception(
+ "The reference %s contains a function name, which doesn't work with run(): %s" % (self.name, self.ref[1]))
+ return self.run_script(*args, **kw)
+
+ def run_py(self, *args, **kw):
+ obj = self.get_object()
+ ## FIXME: catch stdout/stderr?
+ try:
+ result = obj(*args, **kw)
+ except Exception, e:
+ return None, {'exception': e}
+ else:
+ return result, {}
+
+ def get_object(self):
+ if self.ref_type == 'pyscript':
+ filename = self.ref_data[0]
+ name = self.ref_data[1]
+ ns = {
+ '__file__': filename,
+ '__name__': os.path.splitext(os.path.basename(filename))[0],
+ }
+ execfile(filename, ns)
+ ## FIXME: error check:
+ return ns[name]
+ else:
+ modname = self.ref_data[0]
+ name = self.ref_data[1]
+ __import__(modname)
+ mod = sys.modules[modname]
+ return getattr(mod, name)
class Environment(object):
@@ -383,6 +475,7 @@ def __init__(self, app, config=None, env_description=None, env_base=None,
self.env_base = env_base
self.base_python_exe = base_python_exe
self.venv_location = venv_location
+ self.app.environment = self
def run_command(self, command_path, args=None, env=None):
if self.env_base:
View
27 apppkg/init.py
@@ -3,6 +3,8 @@
import argparse
import os
+here = os.path.dirname(os.path.abspath(__file__))
+
parser = argparse.ArgumentParser(
prog='python -c apppkg.init',
description="Create a new apppkg layout",
@@ -19,8 +21,9 @@
TEMPLATE_DIRS = [
'.',
- '%(pkg_name)s/%(pkg_name)s',
+ '%(pkg_name)s-src/%(pkg_name)s',
'vendor',
+ 'bin',
]
TEMPLATE_FILES = {
@@ -29,7 +32,7 @@
platform: python wsgi
name: %(name)s
add_paths:
- - %(pkg_name)s
+ - %(pkg_name)s-src
requires:
pip: requirements.txt
wsgi: %(pkg_name)s.entrypoints:make_app()
@@ -42,7 +45,9 @@
check_environment: %(pkg_name)s.entrypoints:check_environment
""",
- '%(pkg_name)s/%(pkg_name)s/entrypoints.py': """\
+ 'README.txt': open(os.path.join(here, 'readme-layout.txt')).read(),
+
+ '%(pkg_name)s-src/%(pkg_name)s/entrypoints.py': """\
# Each of the functions here is referred to in app.yaml
# They start out simply stubbed out
@@ -80,9 +85,9 @@ def check_environment():
pass
""",
- '%(pkg_name)s/%(pkg_name)s/__init__.py': """\
+ '%(pkg_name)s-src/%(pkg_name)s/__init__.py': """\
""",
- '%(pkg_name)s/sitecustomize.py': """\
+ '%(pkg_name)s-src/sitecustomize.py': """\
# You can put code here that will be run when the process is setup
""",
@@ -97,6 +102,9 @@ def check_environment():
--install-purelib=%%(here)s/vendor/
--install-platlib=%%(here)s/vendor-binary/
--install-scripts=%%(here)s/bin/
+
+script_fixup = apppkg.scriptfixup:fixup
+
""",
'.gitignore': """\
@@ -105,12 +113,12 @@ def check_environment():
'requirements.txt': """\
# You MAY put libraries here that you require.
-# You SHOUlD instead try to use "pip install" to install things into vendor/
+# You SHOULD instead try to use "pip install" to install things into vendor/
# You WILL notice some libraries end up in vendor-binary/ : these are libraries
# that must be built locally. You should put those libraries into this file.
# You are NOT recommended to use "pip freeze" to generate this file, as it will
# include libraries should be present in vendor/
-"""
+""",
}
@@ -119,7 +127,10 @@ def sub(c, vars):
def make_package_name(name):
- return name.lower().replace(' ', '_')
+ name = name.lower().replace(' ', '_').replace('-', '_')
+ if name.endswith('_app'):
+ name = name[:-4]
+ return name
def main():
View
58 apppkg/readme-layout.txt
@@ -0,0 +1,58 @@
+Welcome to your fancy new apppkg setup. This file describes a bit of
+how you can use this layout.
+
+This layout is setup to host your code, but also to help you manage
+libraries and dependencies for your application.
+
+It is intended that you put your "main" application code in
+%(pkg_name)s-src/ - so that %(pkg_name)s-src/%(pkg_name)s/ is the
+package. (The -src is included to distinguish between the directory
+containing the package and the package itself.) You can rearrange
+this if you want. For example:
+
+ $ mkdir src
+ $ mv %(pkg_name)s-src src/%(pkg_name)s
+ # Then edit app.yaml to point to the new location
+
+You'll notice a file %(pkg_name)s-src/sitecustomize.py where you can
+put code that will always be run at startup (before scripts or
+libraries or anything else).
+
+The file app.yaml contains information about your application. We've
+put several examples in there, with stub code in
+%(pkg_name)s-src/%(pkg_name)s/entrypoints.py
+
+For managing your libraries, there is a file .pip.conf which controls
+pip when you run it from within this directory (or a subdirectory).
+
+When used like this, libraries will be installed into vendor/. You
+can (and should!) check this directory into version control. Also
+bin/ will contain scripts. If a library contains binary components
+and is not portable it will instead be installed into vendor-binary/
+(this directory will be created on demand). You should not check this
+directory into version control! Instead the libraries in there should
+be reinstalled on new systems. You can note these libraries in
+requirements.txt
+
+You may be familiar with requirements.txt from other deployment
+systems. For apppkg you should consider it a last resort - vendor/ is
+a safer and simpler system for most libraries. Also note that you can
+ask that instead of installing libraries with pip -r requirements,
+that they be installed with your native packaging system. Do
+something like this in app.yaml:
+
+requires:
+ deb:
+ - python-lxml
+ rpm:
+ - python-lxml
+ requires:
+ - lxml
+
+Systems should use the deb or rpm method if they can, and then only
+use the requires value as confirmation. I especially recommend this
+for database drivers.
+
+How the directory is assembled is up to you. You may want to use a
+script to check things out, or use git submodules, svn externals,
+whatever works for you.
View
37 apppkg/scriptfixup.py
@@ -0,0 +1,37 @@
+"""This is meant to be called specifically with pip install --script-fixup=apppkg.scriptfixup:fixup"""
+
+def fixup(scripts):
+ for req, script in scripts:
+ _fixup_script(script)
+
+TEMPLATE = """\
+#!/usr/bin/env python
+## This is all apppkg boilerplate for activating the environment:
+import os
+base = os.path.dirname(os.path.abspath(__file__))
+# Now we walk up until we can find app.yaml
+old_base = None
+while not os.path.exists(os.path.join(base, 'app.yaml')):
+ old_base = base
+ base = os.path.dirname(base)
+ if base == old_base:
+ raise Exception('Cannot locate app.yaml above script %s' % __file__)
+import apppkg
+app = apppkg.AppPackage(base)
+app.initialize_for_script()
+
+## Here is the normal script:
+
+__CONTENT__
+"""
+
+def _fixup_script(script):
+ fp = open(script)
+ # Skip the #! line:
+ fp.readline()
+ content = fp.read()
+ fp.close()
+ script_content = TEMPLATE.replace('__CONTENT__', content)
+ fp = open(script, 'w')
+ fp.write(script_content)
+ fp.close()

0 comments on commit fb7abc6

Please sign in to comment.