198 changes: 154 additions & 44 deletions charmtools/build/__init__.py

Large diffs are not rendered by default.

75 changes: 52 additions & 23 deletions charmtools/build/fetchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,25 +33,30 @@ class InterfaceFetcher(fetchers.LocalFetcher):
ENVIRON = "INTERFACE_PATH"
OPTIONAL_PREFIX = "juju-relation-"
ENDPOINT = "/api/v1/interface"
NO_LOCAL_LAYERS = False

@classmethod
def can_fetch(cls, url):
# Search local path first, then
# the interface webservice
# Search local path first, then the interface webservice
if url.startswith("{}:".format(cls.NAMESPACE)):
url = url[len(cls.NAMESPACE) + 1:]
search_path = [os.environ.get("JUJU_REPOSITORY", ".")]
cp = os.environ.get(cls.ENVIRON)
if cp:
search_path.extend(cp.split(os.pathsep))
for part in search_path:
p = (path(part) / url).normpath()
if p.exists():
return dict(path=p)

choices = [url]
if url.startswith(cls.OPTIONAL_PREFIX):
choices.append(url[len(cls.OPTIONAL_PREFIX):])
name = url[len(cls.NAMESPACE) + 1:]

if not cls.NO_LOCAL_LAYERS:
prefixed_name = '{}-{}'.format(cls.NAMESPACE, name)
search_path = [os.environ.get("JUJU_REPOSITORY", ".")]
cp = os.environ.get(cls.ENVIRON)
if cp:
search_path.extend(cp.split(os.pathsep))
for part in search_path:
basepath = path(part)
for dirname in (name, prefixed_name):
p = (basepath / dirname).normpath()
if p.exists():
return dict(path=p)

choices = [name]
if name.startswith(cls.OPTIONAL_PREFIX):
choices.append(name[len(cls.OPTIONAL_PREFIX):])
for choice in choices:
uri = "%s%s/%s/" % (
cls.INTERFACE_DOMAIN, cls.ENDPOINT, choice)
Expand All @@ -65,19 +70,43 @@ def can_fetch(cls, url):
return result
return {}

def target(self, dir_):
"""Return a :class:`path` of the directory where the downloaded item
will be located.
:param str dir_: Directory into which the item will be downloaded.
:return: :class:`path`
"""
if hasattr(self, "path"):
return self.path
elif hasattr(self, "repo"):
_, target = self._get_repo_fetcher_and_target(self.repo, dir_)
return target

def _get_repo_fetcher_and_target(self, repo, dir_):
"""Returns a :class:`Fetcher` for ``repo``, and the destination dir
at which the downloaded repo will be created.
:param str repo: The repo url.
:param str dir_: Directory into which the repo will be downloaded.
:return: 2-tuple of (:class:`Fetcher`, :class:`path`)
"""
u = self.url[len(self.NAMESPACE) + 1:]
f = get_fetcher(repo)
if hasattr(f, "repo"):
basename = path(f.repo).name.splitext()[0]
else:
basename = u
return f, path(dir_) / basename

def fetch(self, dir_):
if hasattr(self, "path"):
return super(InterfaceFetcher, self).fetch(dir_)
elif hasattr(self, "repo"):
# use the github fetcher for now
u = self.url[10:]
f = get_fetcher(self.repo)
if hasattr(f, "repo"):
basename = path(f.repo).name.splitext()[0]
else:
basename = u
f, target = self._get_repo_fetcher_and_target(self.repo, dir_)
res = f.fetch(dir_)
target = dir_ / basename
if res != target:
target.rmtree_p()
path(res).rename(target)
Expand Down
47 changes: 31 additions & 16 deletions charmtools/build/tactics.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,10 @@ def trigger(cls, relpath):


class InterfaceCopy(Tactic):
def __init__(self, interface, relation_name, target, config):
def __init__(self, interface, relation_name, role, target, config):
self.interface = interface
self.relation_name = relation_name
self.role = role
self._target = target
self._config = config

Expand All @@ -154,10 +155,6 @@ def __call__(self):
# directory
log.debug("Copying Interface %s: %s",
self.interface.name, self.target)
# Ensure the path exists
if self.target.exists():
# XXX: fix this to do actual updates
return
ignorer = utils.ignore_matcher(self.config.ignores)
for entity, _ in utils.walk(self.interface.directory,
lambda x: True,
Expand Down Expand Up @@ -186,15 +183,23 @@ def sign(self):
return sigs

def lint(self):
impl = self.interface.directory / self.role + '.py'
if not impl.exists():
log.error('Missing implementation for interface role: %s.py', self.role)
return False
valid = True
for entry in self.interface.directory.walkfiles():
if entry.splitext()[1] != ".py":
continue
relpath = entry.relpath(self._target.directory)
target = self._target.directory / relpath
if not target.exists():
continue
return utils.delta_python_dump(entry, target,
from_name=relpath)
unchanged = utils.delta_python_dump(entry, target,
from_name=relpath)
if not unchanged:
valid = False
return valid


class DynamicHookBind(Tactic):
Expand Down Expand Up @@ -295,6 +300,7 @@ def apply_edits(self):
else:
namespace = self.data
for key in dels:
# TODO: Chuck edit this thing
utils.delete_path(key, namespace)
if not self.target_file.parent.exists():
self.target_file.parent.makedirs_p()
Expand All @@ -317,9 +323,11 @@ def load(self, fn):
return yaml.load(fn, Loader=yaml.RoundTripLoader)

def dump(self, data):
yaml.dump(data, self.target_file.open('w'),
Dumper=yaml.RoundTripDumper,
default_flow_style=False)
with open(self.target_file, 'w') as fd:
yaml.dump(data, fd,
Dumper=yaml.RoundTripDumper,
default_flow_style=False,
default_style='"')


class JSONTactic(SerializedTactic):
Expand Down Expand Up @@ -381,6 +389,7 @@ def trigger(cls, relpath):
def read(self):
if not self._read:
super(LayerYAML, self).read()
self.data.setdefault('options', {})
self.schema['properties'] = {
self.current.name: {
'type': 'object',
Expand All @@ -395,8 +404,7 @@ def combine(self, existing):
return self

def lint(self):
if 'options' not in self.data:
return True
self.read()
validator = extend_with_default(jsonschema.Draft4Validator)(self.schema)
valid = True
for error in validator.iter_errors(self.data['options']):
Expand Down Expand Up @@ -494,6 +502,12 @@ class ConfigYAML(YAMLTactic):
FILENAME = "config.yaml"


class ActionsYAML(YAMLTactic):
"""Rule driven actions.yaml generation"""
section = "actions"
FILENAME = "actions.yaml"


class DistYAML(YAMLTactic):
"""Rule driven dist.yaml generation"""
section = "dist"
Expand Down Expand Up @@ -536,7 +550,7 @@ def __call__(self):
"install",
"--user",
"--ignore-installed",
spec), env=localenv).throw_on_error()()
spec), env=localenv).exit_on_error()()
self._tracked = []
# We now manage two classes of explicit mappings
# When python packages are installed into a prefix
Expand Down Expand Up @@ -605,7 +619,7 @@ def _add(self, pip, wheelhouse, *reqs):
utils.Process((pip, 'install',
'--no-binary', ':all:',
'-d', temp_dir) +
reqs).throw_on_error()()
reqs).exit_on_error()()
for wheel in temp_dir.files():
dest = wheelhouse / wheel.basename()
dest.remove_p()
Expand All @@ -619,8 +633,8 @@ def __call__(self, venv=None):
wheelhouse = self.target.directory / 'wheelhouse'
wheelhouse.mkdir_p()
if create_venv:
utils.Process(('virtualenv', '--python', 'python3', venv)).throw_on_error()()
utils.Process((pip, 'install', '-U', 'pip', 'wheel')).throw_on_error()()
utils.Process(('virtualenv', '--python', 'python3', venv)).exit_on_error()()
utils.Process((pip, 'install', '-U', 'pip', 'wheel')).exit_on_error()()
for tactic in self.previous:
tactic(venv)
self._add(pip, wheelhouse, '-r', self.entity)
Expand Down Expand Up @@ -679,6 +693,7 @@ def set_defaults(validator, properties, instance, schema):
ResourcesYAML,
MetadataYAML,
ConfigYAML,
ActionsYAML,
LayerYAML,
CopyTactic
]
83 changes: 22 additions & 61 deletions charmtools/bundles.py
Original file line number Diff line number Diff line change
@@ -1,87 +1,53 @@
import glob
import json
import os
import re
import yaml

from linter import Linter
from charmworldlib import bundle as cw_bundle
import jujubundlelib.validation


charm_url_includes_id = re.compile(r'-\d+$').search


class BundleLinter(Linter):
def validate(self, data, name=None):
"""Validate the bundle.
def validate(self, data):
"""Supplement jujubundlelib validation with some extra checks.
Tests:
* No series set and not inheriting,
* Position annotations give for each service.
"""
leader = '%s: ' % name if name else ''
if 'series' not in data and 'inherits' not in data:
self.info("%sNo series defined" % leader)
self.info("No series defined")

if 'services' in data:
for svc, sdata in data['services'].items():
if 'annotations' not in sdata:
self.warn('%s%s: No annotations found, will render '
'poorly in GUI' % (leader, svc))
if not charm_url_includes_id(sdata['charm']):
self.warn('%s: No annotations found, will render '
'poorly in GUI' % svc)
if ('charm' in sdata and
not charm_url_includes_id(sdata['charm'] or '')):
self.warn(
'%s%s: charm URL should include a revision' % (
leader, svc))

'%s: charm URL should include a revision' % svc)
else:
if 'inherits' not in data:
self.err("%sNo services defined" % leader)
return
self.err("No services defined")

def local_proof(self, bundle):
def proof(self, bundle):
data = bundle.bundle_file()
if not bundle.is_v4(data):
self.err(
'This bundle format is no longer supported. See '
'https://jujucharms.com/docs/stable/charms-bundles '
'for the supported format.')
return

readmes = glob.glob(os.path.join(bundle.bundle_path, 'README*'))
if len(readmes) < 1:
self.warn('No readme file found')

if bundle.is_v4(data):
self.validate(data)
else:
for name, bdata in data.items():
if name == 'envExport':
self.warn('envExport is the default export name. Please '
'use a unique name')
self.validate(bdata, name)

def remote_proof(self, bundle, server, port, secure):
data = bundle.bundle_file()
if bundle.is_v4(data):
# use jujubundlelib in lieu of deprecated API
errors = jujubundlelib.validation.validate(data)
for error in errors:
self.err(error)
return

if server is not None or port is not None:
# Use the user-specified overrides for the remote server.
bundles = cw_bundle.Bundles(server=server, port=port,
secure=secure)
else:
# Otherwise use the defaults.
bundles = cw_bundle.Bundles()

proof_output = bundles.proof(data)

if self.debug:
print json.dumps(proof_output, 2)

for key, emitter in (('error_messages', self.err),
('warning_messages', self.warn)):
if key in proof_output:
for message in proof_output[key]:
emitter(message)
errors = jujubundlelib.validation.validate(data)
for error in errors:
self.err(error)
self.validate(data)


class Bundle(object):
Expand Down Expand Up @@ -122,14 +88,9 @@ def bundle_file(self, parse=True):

raise Exception('No bundle.json or bundle.yaml file found')

def proof(self, remote=True, server=None, port=None, secure=True):
def proof(self):
lint = BundleLinter(self.debug)
lint.local_proof(self)
if remote:
lint.remote_proof(self, server, port, secure)
else:
lint.info('No remote checks performed')

lint.proof(self)
return lint.lint, lint.exit_code

def promulgate(self):
Expand Down
169 changes: 166 additions & 3 deletions charmtools/charms.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
'summary',
'maintainer',
'maintainers',
'min-juju-version',
'description',
'categories',
'subordinate',
Expand All @@ -25,7 +26,9 @@
'format',
'peers',
'tags',
'series',
'storage',
'extra-bindings',
]

KNOWN_RELATION_KEYS = ['interface', 'scope', 'limit', 'optional']
Expand Down Expand Up @@ -57,13 +60,32 @@ class RelationError(Exception):


class CharmLinter(Linter):

# _WINDOWS_HOOKS_EXTS is the list of possible extensions for hooks
# on Windows. File extensions must be present in Windows ad thus
# we must specially check for them when linting the hooks.
_WINDOWS_HOOKS_EXTS = [".ps1", ".cmd", ".bat", ".exe"]

def check_hook(self, hook, hooks_path, recommended=False):
hook_path = os.path.join(hooks_path, hook)
ispscharm = False # flag to indicate whether PowerShell charm or not.

# iterate through the possible hook-extension
# combinations and find the right one:
for path in [hook_path + ext for ext in self._WINDOWS_HOOKS_EXTS]:
if os.path.isfile(path):
hook_path = path
ispscharm = True
break

try:
mode = os.stat(hook_path)[ST_MODE]
if not mode & S_IXUSR:

# NOTE: hooks on Windows are judged as executable depending on
# their extension; not their mode.
if (not mode & S_IXUSR) and not ispscharm:
self.info(hook + " not executable")

with open(hook_path, 'r') as hook_file:
count = 0
for line in hook_file:
Expand Down Expand Up @@ -138,7 +160,7 @@ def check_config_file(self, charm_path):
try:
with open(config_path) as config_file:
config = yaml.safe_load(config_file.read())
except Exception, error:
except Exception as error:
self.err('Cannot parse config.yaml: %s' % error)
return
if not isinstance(config, dict):
Expand Down Expand Up @@ -221,7 +243,7 @@ def __init__(self, path):
def is_charm(self):
return os.path.isfile(os.path.join(self.charm_path, 'metadata.yaml'))

def proof(self, remote=True, **kw):
def proof(self):
lint = CharmLinter()
charm_name = self.charm_path
if os.path.isdir(charm_name):
Expand All @@ -235,7 +257,9 @@ def proof(self, remote=True, **kw):
return lint.lint, lint.exit_code

hooks_path = os.path.join(charm_path, 'hooks')
actions_path = os.path.join(charm_path, 'actions')
yaml_path = os.path.join(charm_path, 'metadata.yaml')
actions_yaml_file = os.path.join(charm_path, 'actions.yaml')
try:
yamlfile = open(yaml_path, 'r')
try:
Expand Down Expand Up @@ -265,6 +289,10 @@ def proof(self, remote=True, **kw):
validate_maintainer(charm, lint)
validate_categories_and_tags(charm, lint)
validate_storage(charm, lint)
validate_series(charm, lint)
validate_min_juju_version(charm, lint)
validate_extra_bindings(charm, lint)
validate_payloads(charm, lint)

if not os.path.exists(os.path.join(charm_path, 'icon.svg')):
lint.info("No icon.svg file.")
Expand Down Expand Up @@ -384,6 +412,15 @@ def proof(self, remote=True, **kw):
lint.check_hook('config-changed', hooks_path, recommended=True)
else:
lint.check_hook('config-changed', hooks_path)

if os.path.exists(actions_yaml_file):
with open(actions_yaml_file) as f:
try:
actions = yaml.safe_load(f.read())
except Exception as e:
lint.crit('cannot parse ' + actions_yaml_file + ":" + str(e))
validate_actions(actions, actions_path, lint)

except IOError:
lint.err("could not find metadata file for " + charm_name)
lint.exit_code = -1
Expand Down Expand Up @@ -479,6 +516,76 @@ def schema_type(self, **kw):
)


class PayloadItem(colander.MappingSchema):
def schema_type(self, **kw):
return colander.Mapping(unknown='raise')

type_ = colander.SchemaNode(
colander.String(),
validator=colander.OneOf(['kvm', 'docker']),
name='type',
)


def validate_extra_bindings(charm, linter):
"""Validate extra-bindings in charm metadata.
:param charm: dict of charm metadata parsed from metadata.yaml
:param linter: :class:`CharmLinter` object to which info/warning/error
messages will be written
"""
if 'extra-bindings' not in charm:
return

if not isinstance(charm['extra-bindings'], dict):
linter.err('extra-bindings: must be a dictionary')


def validate_min_juju_version(charm, linter):
"""Validate min-juju-version in charm metadata.
Must match the regex and be 2.0.0 or greater.
:param charm: dict of charm metadata parsed from metadata.yaml
:param linter: :class:`CharmLinter` object to which info/warning/error
messages will be written
"""
if 'min-juju-version' not in charm:
return

pattern = r'^(\d{1,9})\.(\d{1,9})(\.|-(\w+))(\d{1,9})(\.\d{1,9})?$'
match = re.match(pattern, charm['min-juju-version'])
if not match:
linter.err('min-juju-version: invalid format, try X.Y.Z')
return

if int(match.group(1)) < 2:
linter.err(
'min-juju-version: invalid version, must be 2.0.0 or greater')


def validate_series(charm, linter):
"""Validate supported series list in charm metadata.
We don't validate the actual series names because:
1. `charm push` does that anyway
2. our list of valid series would be constantly falling out-of-date
:param charm: dict of charm metadata parsed from metadata.yaml
:param linter: :class:`CharmLinter` object to which info/warning/error
messages will be written
"""
if 'series' not in charm:
return

if not isinstance(charm['series'], list):
linter.err('series: must be a list of series names')


def validate_storage(charm, linter):
"""Validate storage configuration in charm metadata.
Expand Down Expand Up @@ -506,6 +613,62 @@ def validate_storage(charm, linter):
linter.err('storage.{}: {}'.format(k, v))


def validate_payloads(charm, linter):
"""Validate paylaod configuration in charm metadata.
:param charm: dict of charm metadata parsed from metadata.yaml
:param linter: :class:`CharmLinter` object to which info/warning/error
messages will be written
"""
if 'payloads' not in charm:
return

if (not isinstance(charm['payloads'], dict) or
not charm['payloads']):
linter.err('payloads: must be a dictionary of payload definitions')
return

schema = colander.SchemaNode(colander.Mapping())
for payload_def in charm['payloads']:
schema.add(PayloadItem(name=payload_def))

try:
schema.deserialize(charm['payloads'])
except colander.Invalid as e:
for k, v in e.asdict().items():
linter.err('payloads.{}: {}'.format(k, v))


def validate_actions(actions, action_hooks, linter):
"""Validate actions in a charm.
:param actions: dict of charm actions parsed from actions.yaml
:param action_hooks: path of charm's /actions/ directory
:param linter: :class:`CharmLinter` object to which info/warning/error
messages will be written
"""

if not actions:
return

if not isinstance(actions, dict):
linter.err('actions: must be a dictionary of json schemas')
return

# TODO: Schema validation
for k in actions:
if k.startswith('juju'):
linter.err('actions.{}: juju is a reserved namespace'.format(k))
continue
h = os.path.join(action_hooks, k)
if not os.path.isfile(h):
linter.warn('actions.{0}: actions/{0} does not exist'.format(k))
elif not os.access(h, os.X_OK):
linter.err('actions.{0}: actions/{0} is not executable'.format(k))


def validate_maintainer(charm, linter):
"""Validate maintainer info in charm metadata.
Expand Down
4 changes: 3 additions & 1 deletion charmtools/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ def parser_defaults(parser):


def usage(exit_code=0, bundle=False):
sys.stderr.write('usage: %s subcommand\n' % os.path.basename(sys.argv[0]))
sys.stderr.write(
'Get help for a charm subcommand\n\n'
'usage: %s subcommand\n' % os.path.basename(sys.argv[0]))
subs = subcommands(os.path.dirname(os.path.realpath(__file__)))
sys.stderr.write('\n Available subcommands are:\n ')
sys.stderr.write('\n '.join(subs))
Expand Down
10 changes: 8 additions & 2 deletions charmtools/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@
CharmGeneratorException,
get_installed_templates,
)
from . import utils

log = logging.getLogger(__name__)

DEFAULT_TEMPLATE = 'python'
DEFAULT_TEMPLATE = 'reactive-python'


def setup_parser():
parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser(
description='create a new charm')

parser.add_argument(
'charmname',
Expand All @@ -60,6 +62,7 @@ def setup_parser():
help='Print debug information',
action='store_true', default=False,
)
utils.add_plugin_description(parser)

return parser

Expand All @@ -86,6 +89,9 @@ def main():
"Using default charm template (%s). To select a different "
"template, use the -t option.", DEFAULT_TEMPLATE)
args.template = DEFAULT_TEMPLATE
elif args.template not in get_installed_templates():
raise Exception("No template available for '%s'. Available templates "
"may be listed by running 'charm create --help'.")

generator = CharmGenerator(args)
try:
Expand Down
203 changes: 116 additions & 87 deletions charmtools/fetchers.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import errno
import logging
import os
import re
Expand All @@ -9,7 +10,6 @@
import requests
import yaml

from charmworldlib.bundle import Bundle

log = logging.getLogger(__name__)

Expand All @@ -23,6 +23,14 @@ def get(*args, **kw):
return requests.get(*args, **kw)


def is_int(string):
try:
int(string)
return True
except ValueError:
return False


def rename(dir_):
"""If ``dir_`` is a charm directory, rename it to match the
charm name, otherwise do nothing.
Expand Down Expand Up @@ -51,6 +59,41 @@ def rename(dir_):
return new_dir


def extract_archive(archive, dir_):
"""Extract zip archive at filesystem path ``archive`` into directory
``dir_`` and return the full path to the directory containing the
extracted archive.
"""
tempdir = tempfile.mkdtemp(dir=dir_)
log.debug("Extracting %s to %s", archive, tempdir)
# Can't extract with python due to bug that drops file
# permissions: http://bugs.python.org/issue15795
# In particular, it's important that executable test files in the
# archive remain executable, otherwise the tests won't be run.
# Instead we use a shell equivalent of the following:
# archive = zipfile.ZipFile(archive, 'r')
# archive.extractall(tempdir)
check_call('unzip {} -d {}'.format(archive, tempdir))
return tempdir


def download_file(url, dir_):
"""Download file at ``url`` into directory ``dir_`` and return the full
path to the downloaded file.
"""
_, filename = tempfile.mkstemp(dir=dir_)
log.debug("Downloading %s", url)
r = get(url, stream=True)
with open(filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return filename


class Fetcher(object):
def __init__(self, url, **kw):
self.revision = ''
Expand Down Expand Up @@ -79,7 +122,7 @@ def get_revision(self, dir_):

class BzrFetcher(Fetcher):
MATCH = re.compile(r"""
^(lp:|launchpad:|https?://((code|www)\.)?launchpad.net/)
^(lp:|launchpad:|https?://((code|www)\.)?launchpad.net/|bzr\+ssh://[^/]+/)
(?P<repo>[^@]*)(@(?P<revision>.*))?$
""", re.VERBOSE)

Expand Down Expand Up @@ -134,7 +177,7 @@ def fetch(self, dir_):

class GithubFetcher(Fetcher):
MATCH = re.compile(r"""
^(gh:|github:|https?://(www\.)?github.com/)
^(gh:|github:|https?://(www\.)?github.com/|git@github.com:)
(?P<repo>[^@]*)(@(?P<revision>.*))?$
""", re.VERBOSE)

Expand All @@ -147,6 +190,22 @@ def fetch(self, dir_):
return rename(dir_)


class GitFetcher(Fetcher):
"""Generic git fetcher.
Matches any url that starts with "git" or ends with ".git".
"""
MATCH = re.compile(r"""
^(?P<repo>git.*|.*\.git)?$
""", re.VERBOSE)

def fetch(self, dir_):
dir_ = tempfile.mkdtemp(dir=dir_)
git('clone {} {}'.format(self.repo, dir_))
return rename(dir_)


class BitbucketFetcher(Fetcher):
MATCH = re.compile(r"""
^(bb:|bitbucket:|https?://(www\.)?bitbucket.org/)
Expand Down Expand Up @@ -189,101 +248,64 @@ def fetch(self, dir_):
return dst


class StoreCharm(object):
STORE_URL = 'https://store.juju.ubuntu.com/charm-info'

def __init__(self, name):
self.name = name
self.data = self.fetch()

def __getattr__(self, key):
return self.data[key]

def fetch(self):
params = {
'stats': 0,
'charms': self.name,
}
r = get(self.STORE_URL, params=params).json()
charm_data = r[self.name]
if 'errors' in charm_data:
raise FetchError(
'Error retrieving "{}" from charm store: {}'.format(
self.name, '; '.join(charm_data['errors']))
)
return charm_data


class CharmstoreDownloader(Fetcher):
"""Downloads and extracts a charm archive from the charm store.
"""
MATCH = re.compile(r"""
^cs:(?P<charm>.*)$
^cs:(?P<entity>.*)$
""", re.VERBOSE)

STORE_URL = 'https://store.juju.ubuntu.com/charm/'
STORE_URL = 'https://api.jujucharms.com/charmstore/v4/{}'
ARCHIVE_URL = STORE_URL + '/archive'
REVISION_URL = STORE_URL + '/meta/id-revision'

def __init__(self, *args, **kw):
super(CharmstoreDownloader, self).__init__(*args, **kw)
self.charm = StoreCharm(self.charm)

def fetch(self, dir_):
url = self.charm.data['canonical-url'][len('cs:'):]
url = self.STORE_URL + url
archive = self.download_file(url, dir_)
charm_dir = self.extract_archive(archive, dir_)
return rename(charm_dir)

def extract_archive(self, archive, dir_):
tempdir = tempfile.mkdtemp(dir=dir_)
log.debug("Extracting %s to %s", archive, tempdir)
# Can't extract with python due to bug that drops file
# permissions: http://bugs.python.org/issue15795
# In particular, it's important that executable test files in the
# archive remain executable, otherwise the tests won't be run.
# Instead we use a shell equivalent of the following:
# archive = zipfile.ZipFile(archive, 'r')
# archive.extractall(tempdir)
check_call('unzip {} -d {}'.format(archive, tempdir))
return tempdir

def download_file(self, url, dir_):
_, filename = tempfile.mkstemp(dir=dir_)
log.debug("Downloading %s", url)
r = get(url, stream=True)
with open(filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return filename
url = self.ARCHIVE_URL.format(self.entity)
archive = download_file(url, dir_)
entity_dir = extract_archive(archive, dir_)
return rename(entity_dir)

def get_revision(self, dir_):
return self.charm.revision
url = self.REVISION_URL.format(self.entity)
return get(url).json()['Revision']


class BundleDownloader(Fetcher):
class BundleDownloader(CharmstoreDownloader):
MATCH = re.compile(r"""
^bundle:(?P<bundle>.*)$
^bundle:(?P<entity>.*)$
""", re.VERBOSE)

def fetch(self, dir_):
url = Bundle(self.bundle).deployer_file_url
bundle_dir = self.download_file(url, dir_)
return bundle_dir

def download_file(self, url, dir_):
bundle_dir = tempfile.mkdtemp(dir=dir_)
bundle_file = os.path.join(bundle_dir, 'bundles.yaml')
log.debug("Downloading %s to %s", url, bundle_file)
r = get(url, stream=True)
with open(bundle_file, 'w') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return bundle_dir
def __init__(self, *args, **kw):
super(BundleDownloader, self).__init__(*args, **kw)
self.entity = normalize_bundle_name(self.entity)

def get_revision(self, dir_):
return Bundle(self.bundle).basket_revision

def normalize_bundle_name(bundle_name):
"""Convert old-style bundle name to new format.
Example:
~charmers/mediawiki/6/single -> ~charmers/mediawiki-single-6
(for more examples see tests)
"""
owner, bundle = None, bundle_name
if bundle.startswith('~'):
owner, bundle = bundle.split('/', 1)
bundle_parts = bundle.split('/')
if len(bundle_parts) == 3 and is_int(bundle_parts[1]):
bundle_parts = [
bundle_parts[0],
bundle_parts[2],
bundle_parts[1]]
bundle = '-'.join(bundle_parts)
if owner:
bundle = '/'.join((owner, bundle))
return bundle


def bzr(cmd, **kw):
Expand All @@ -308,12 +330,18 @@ class FetchError(Exception):

def check_output(cmd, **kw):
args = shlex.split(cmd)
p = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kw
)
try:
p = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kw
)
except OSError as e:
msg = 'Unable to run "%s": %s' % (args[0], e.strerror)
if e.errno == errno.ENOENT:
msg += '\nPlease install "%s" and try again' % args[0]
raise FetchError(msg)
out, _ = p.communicate()
if p.returncode != 0:
raise FetchError(out)
Expand All @@ -330,6 +358,7 @@ def check_output(cmd, **kw):
CharmstoreDownloader,
BundleDownloader,
LaunchpadGitFetcher,
GitFetcher,
]


Expand Down
23 changes: 17 additions & 6 deletions charmtools/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,26 @@

from cli import parser_defaults
from charms import Charm
from charmworldlib.charm import Charms
from charmstore import CharmStore
from charmstore.error import CharmNotFound
from . import utils

TPL_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'templates')
CHARM_TPL = os.path.join(TPL_DIR, 'charm')


def graph(interface, endpoint, series='trusty'):
matches = {'requires': 'provides', 'provides': 'requires'}
c = Charms()
charms = c.search({matches[endpoint]: interface, 'series': series})
matches = {
'requires': 'provides',
'provides': 'requires',
}
match = matches[endpoint]
c = CharmStore()
try:
charms = getattr(c, match)(interface)
except CharmNotFound:
return None
charms = [c for c in charms if c.series == series]
if charms:
return charms[0]
else:
Expand Down Expand Up @@ -100,16 +110,17 @@ def tests(charm_dir, is_bundle=False, debug=False, series='trusty'):

def parser(args=None):
parser = argparse.ArgumentParser(
description='Builds portions of a charm or bundle')
description='add icon, readme, or tests to a charm')
parser.add_argument('subcommand', choices=['tests', 'readme', 'icon'],
help='Which type of generator to run')
utils.add_plugin_description(parser)
parser = parser_defaults(parser)
return parser.parse_known_args(args)


def tests_parser(args):
# This bites, need an argparser experter
parser = argparse.ArgumentParser(description="Add tests to a charm")
parser = argparse.ArgumentParser(description="add tests to a charm")
parser.add_argument('--series', '-s', default='trusty',
help='Series for the generated test')
return parser.parse_args(args)
Expand Down
86 changes: 0 additions & 86 deletions charmtools/get.py

This file was deleted.

69 changes: 0 additions & 69 deletions charmtools/getall.py

This file was deleted.

56 changes: 0 additions & 56 deletions charmtools/info.py

This file was deleted.

245 changes: 0 additions & 245 deletions charmtools/promulgate.py

This file was deleted.

23 changes: 7 additions & 16 deletions charmtools/proof.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,22 @@
from bundles import Bundle
from charms import Charm
from cli import parser_defaults
from charmtools import utils


def get_args(args=None):
parser = argparse.ArgumentParser(
description='Performs static analysis on charms and bundles')
parser.add_argument('-n', '--offline', action='store_false',
help='Only perform offline proofing')
parser.add_argument('--server', default=None,
help=argparse.SUPPRESS)
parser.add_argument('--port', default=None, type=int,
help=argparse.SUPPRESS)
parser.add_argument('--secure', action='store_true',
help=argparse.SUPPRESS)
description='perform static analysis on a charm or bundle')
parser.add_argument('charm_name', nargs='?', default=os.getcwd(),
help='path of charm dir to check. Defaults to PWD')
utils.add_plugin_description(parser)
parser = parser_defaults(parser)
args = parser.parse_args(args)

return args


def proof(path, is_bundle, with_remote, debug, server, port, secure):
def proof(path, is_bundle, debug):
path = os.path.abspath(path)
if not is_bundle:
try:
Expand All @@ -59,18 +53,15 @@ def proof(path, is_bundle, with_remote, debug, server, port, secure):
except Exception as e:
return ["FATAL: %s" % e.message], 200

lint, err_code = c.proof(
remote=with_remote, server=server, port=port, secure=secure)
lint, err_code = c.proof()
return lint, err_code


def main():
args_ = get_args()
lint, exit_code = proof(args_.charm_name, args_.bundle, args_.offline,
args_.debug, args_.server, args_.port,
args_.secure)
lint, exit_code = proof(args_.charm_name, args_.bundle, args_.debug)
if lint:
print "\n".join(lint)
print("\n".join(lint))
sys.exit(exit_code)


Expand Down
235 changes: 235 additions & 0 deletions charmtools/pullsource.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
#!/usr/bin/python
#
# pull-source - Fetch source for charm, layers, and interfaces
#
# Copyright (C) 2016 Canonical Ltd.
# Author: Tim Van Steenburgh <tvansteenburgh@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

"""download the source code for a charm, layer, or interface.
The item to download can be specified using any of the following forms:
- [cs:]charm
- [cs:]series/charm
- [cs:]~user/charm
- [cs:]~user/series/charm
- layer:layer-name
- interface:interface-name
If the item is a layered charm, and the top layer of the charm has a repo
key in layer.yaml, the top layer repo will be cloned. Otherwise, the charm
archive will be downloaded and extracted from the charm store.
If a download directory is not specified, the following environment vars
will be used to determine the download location:
- For charms, $JUJU_REPOSITORY
- For layers, $LAYER_PATH
- For interfaces, $INTERFACE_PATH
If a download location can not be determined from environment variables,
the current working directory will be used.
The download is aborted if the destination directory already exists.
"""

import argparse
import atexit
import logging
import os
import shutil
import sys
import tempfile
import textwrap

import yaml

from . import utils
from .build import fetchers
from fetchers import (
CharmstoreDownloader,
FETCHERS,
get,
)


log = logging.getLogger(__name__)

LAYER_PREFIX = 'layer:'
INTERFACE_PREFIX = 'interface:'
CHARM_PREFIX = 'cs:'

ERR_DIR_EXISTS = "Aborting, destination directory exists"


class CharmstoreRepoDownloader(CharmstoreDownloader):
"""Clones a charm's bzr repo.
If the a bzr repo is not set, falls back to
:class:`fetchers.CharmstoreDownloader`.
"""
EXTRA_INFO_URL = CharmstoreDownloader.STORE_URL + '/meta/extra-info'

def fetch(self, dir_):
url = self.EXTRA_INFO_URL.format(self.entity)
repo_url = get(url).json().get('bzr-url')
if repo_url:
try:
fetcher = fetchers.get_fetcher(repo_url)
except fetchers.FetchError:
log.debug(
"No fetcher for %s, downloading from charmstore",
repo_url)
return super(CharmstoreRepoDownloader, self).fetch(dir_)
else:
return fetcher.fetch(dir_)
return super(CharmstoreRepoDownloader, self).fetch(dir_)

FETCHERS.insert(0, CharmstoreRepoDownloader)


class CharmstoreLayerDownloader(CharmstoreRepoDownloader):
"""Clones the repo containing the top layer of a charm.
If the charm is not a layered charm, or the repo for the
top layer can not be determined, falls back to using
:class:`CharmstoreRepoDownloader`.
"""
LAYER_CONFIGS = ['layer.yaml', 'composer.yaml']

def fetch(self, dir_):
for cfg in self.LAYER_CONFIGS:
url = '{}/{}'.format(
self.ARCHIVE_URL.format(self.entity), cfg)
result = get(url)
if not result.ok:
continue
repo_url = yaml.safe_load(result.text).get('repo')
if not repo_url:
continue
try:
fetcher = fetchers.get_fetcher(repo_url)
except fetchers.FetchError:
log.debug(
'Charm %s has a repo set in %s, but no fetcher could '
'be found for the repo (%s).', self.entity, cfg, repo_url)
break
else:
return fetcher.fetch(dir_)
return super(CharmstoreLayerDownloader, self).fetch(dir_)

FETCHERS.insert(0, CharmstoreLayerDownloader)


def download_item(item, dir_):
series_dir = None

if item.startswith(LAYER_PREFIX):
dir_ = dir_ or os.environ.get('LAYER_PATH')
name = item[len(LAYER_PREFIX):]
elif item.startswith(INTERFACE_PREFIX):
dir_ = dir_ or os.environ.get('INTERFACE_PATH')
name = item[len(INTERFACE_PREFIX):]
else:
dir_ = dir_ or os.environ.get('JUJU_REPOSITORY')
if not item.startswith(CHARM_PREFIX):
item = CHARM_PREFIX + item

url_parts = item[len(CHARM_PREFIX):].split('/')
name = url_parts[-1]
if len(url_parts) == 2 and not url_parts[0].startswith('~'):
series_dir = url_parts[0]
elif len(url_parts) == 3:
series_dir = url_parts[1]

dir_ = dir_ or os.getcwd()
dir_ = os.path.abspath(os.path.expanduser(dir_))

# Create series dir if we need to
if series_dir:
series_path = os.path.join(dir_, series_dir)
if not os.path.exists(series_path):
os.mkdir(series_path)
dir_ = series_path

# Abort if destination dir already exists
final_dest_dir = os.path.join(dir_, name)
if os.path.exists(final_dest_dir):
return "{}: {}".format(ERR_DIR_EXISTS, final_dest_dir)

# Create tempdir for initial download
tempdir = tempfile.mkdtemp()
atexit.register(shutil.rmtree, tempdir)
try:
# Download the item
fetcher = fetchers.get_fetcher(item)
download_dir = fetcher.fetch(tempdir)
except fetchers.FetchError:
return "Can't find source for {}".format(item)

# Copy download dir to final destination dir
shutil.copytree(download_dir, final_dest_dir)
print('Downloaded {} to {}'.format(item, final_dest_dir))


def setup_parser():
parser = argparse.ArgumentParser(
prog='charm pull-source',
description=textwrap.dedent(__doc__),
formatter_class=argparse.RawDescriptionHelpFormatter,
)

parser.add_argument(
'item',
help='Name of the charm, layer, or interface to download.'
)
parser.add_argument(
'dir', nargs='?',
help='Directory in which to place the downloaded source.',
)
parser.add_argument(
'-v', '--verbose',
help='Show verbose output',
action='store_true', default=False,
)
utils.add_plugin_description(parser)

return parser


def main():
parser = setup_parser()
args = parser.parse_args()

if args.verbose:
logging.basicConfig(
format='%(levelname)s %(filename)s: %(message)s',
level=logging.DEBUG,
)
else:
logging.basicConfig(
format='%(levelname)s: %(message)s',
level=logging.WARN,
)

return download_item(args.item, args.dir)


if __name__ == "__main__":
sys.exit(main())
97 changes: 97 additions & 0 deletions charmtools/repofinder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import re
import subprocess

from collections import namedtuple

from . import utils


def get_recommended_repo(path):
"""Given vcs directory ``path``, returns the url from which the repo
can be cloned.
For git, an 'upstream' remote will be preferred over 'origin'.
For bzr, the :parent: branch will be preferred.
For hg, the 'default' alias will be preferred.
Returns None if the directory is not a repo, or a remote url can not
be determined.
:param path: A :class:`path.path` to a directory
:return: A url string, or None
"""

Command = namedtuple("Command", "args parse")
cmds = [
Command(['git', 'remote', '-v'], _parse_git),
Command(['bzr', 'info'], _parse_bzr),
Command(['hg', 'paths'], _parse_hg),
]

if not path.exists():
return None

with utils.cd(str(path)):
for cmd in cmds:
try:
output = subprocess.check_output(cmd.args)
if output:
repo = cmd.parse(output)
if repo:
return repo
except (subprocess.CalledProcessError, OSError):
continue


def _parse_git(txt):
pat = re.compile(
r'(?P<name>\S+)\s+(?P<url>\S+)\s+\((?P<type>[^\)]+)\)')
urls = {}
for line in txt.split('\n'):
match = pat.search(line)
if match:
d = match.groupdict()
if d['name'] == 'upstream' and d['type'] == 'fetch':
return d['url'].strip()
elif d['type'] == 'fetch':
urls[d['name']] = d['url'].strip()

if 'origin' in urls:
return urls['origin']

for url in urls.values():
return url


def _parse_bzr(txt):
branch_types = ['parent', 'push', 'submit']
pat = re.compile(
r'(?P<branch_type>({})) branch: (?P<url>.*)'.format(
'|'.join(branch_types)))
matches = {}
for line in txt.split('\n'):
match = pat.search(line)
if match:
d = match.groupdict()
matches[d['branch_type']] = d['url'].strip()
if not matches:
return
for typ in branch_types:
url = matches.get(typ)
if url:
return url


def _parse_hg(txt):
pat = re.compile(r'(?P<name>[^\s]+) = (?P<url>.*)')
urls = []
for line in txt.split('\n'):
match = pat.search(line)
if match:
d = match.groupdict()
if d['name'] == 'default':
return d['url'].strip()
else:
urls.append(d['url'].strip())
return urls[0] if urls else None
118 changes: 0 additions & 118 deletions charmtools/review.py

This file was deleted.

156 changes: 0 additions & 156 deletions charmtools/review_queue.py

This file was deleted.

35 changes: 0 additions & 35 deletions charmtools/search.py

This file was deleted.

191 changes: 0 additions & 191 deletions charmtools/subscribers.py

This file was deleted.

19 changes: 19 additions & 0 deletions charmtools/templates/powershell/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/python
#
# Copyright (C) 2016 Canonical Ltd.
# Copyright (C) 2016 Cloudbase Solutions SRL
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from .template import PowerShellCharmTemplate # noqa
61 changes: 61 additions & 0 deletions charmtools/templates/powershell/template.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/usr/bin/python
#
# Copyright (C) 2016 Canonical Ltd.
# Copyright (C) 2016 Cloudbase Solutions SRL
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import os
import os.path as path
import shutil
import subprocess

from charmtools.generators import CharmTemplate


class PowerShellCharmTemplate(CharmTemplate):
""" CharmTemplate specific to PowerShell charms. """

# _EXTRA_FILES is the list of names of files present in the git repo
# we don't want transferred over to the charm template:
_EXTRA_FILES = ["README.md", ".git", ".gitmodules"]

_TEMPLATE_URL = "https://github.com/cloudbase/windows-charms-boilerplate"

def __init__(self):
self.skip_parsing += ["*.ps1", "*.psm1"]

def create_charm(self, config, output_dir):
cmd = "git clone --recursive {} {}".format(
self._TEMPLATE_URL, output_dir
)

try:
subprocess.check_call(cmd.split())
except OSError as e:
raise Exception(
"The below error has ocurred whilst attempting to clone"
"the powershell charm template. Please make sure you have"
"git installed on your system.\n" + e
)

# iterate and remove all the unwanted files from the git repo:
for item in [path.join(output_dir, i) for i in self._EXTRA_FILES]:
if not path.exists(item):
continue

if path.isdir(item) and not path.islink(item):
shutil.rmtree(item)
else:
os.remove(item)
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def _is_pyfile(path):
def ensure_init(path):
'''
ensure directories leading up to path are importable, omitting
parent directory, eg path='/hooks/helpers/foo'/:
parent directory, eg path='/hooks/helpers/foo/':
hooks/
hooks/helpers/__init__.py
hooks/helpers/foo/__init__.py
Expand Down
19 changes: 19 additions & 0 deletions charmtools/templates/reactive_bash/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/python
#
# Copyright (C) 2014 Canonical Ltd.
# Author: Clint Byrum <clint.byrum@canonical.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from .template import ReactiveBashCharmTemplate # noqa
98 changes: 98 additions & 0 deletions charmtools/templates/reactive_bash/template.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/usr/bin/python
#
# Copyright (C) 2014 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import logging
import os
import os.path as path
import time
import shutil
import subprocess
import tempfile

from Cheetah.Template import Template
from stat import ST_MODE

from charmtools.generators import (
CharmTemplate,
)

log = logging.getLogger(__name__)


class ReactiveBashCharmTemplate(CharmTemplate):
"""Creates a reactive, layered bash-based charm"""

# _EXTRA_FILES is the list of names of files present in the git repo
# we don't want transferred over to the charm template:
_EXTRA_FILES = ["README.md", ".git", ".gitmodules"]

_TEMPLATE_URL = "https://github.com/juju-solutions/template-reactive-bash"

def create_charm(self, config, output_dir):
self._clone_template(config, output_dir)

for root, dirs, files in os.walk(output_dir):
for outfile in files:
if self.skip_template(outfile):
continue

self._template_file(config, path.join(root, outfile))

def _template_file(self, config, outfile):
if path.islink(outfile):
return

mode = os.stat(outfile)[ST_MODE]
t = Template(file=outfile, searchList=(config))
o = tempfile.NamedTemporaryFile(
dir=path.dirname(outfile), delete=False)
os.chmod(o.name, mode)
o.write(str(t))
o.close()
backupname = outfile + str(time.time())
os.rename(outfile, backupname)
os.rename(o.name, outfile)
os.unlink(backupname)

def _clone_template(self, config, output_dir):
cmd = "git clone --recursive {} {}".format(
self._TEMPLATE_URL, output_dir
)

try:
subprocess.check_call(cmd.split())
except OSError as e:
raise Exception(
"The below error has occurred whilst attempting to clone"
"the charm template. Please make sure you have git"
"installed on your system.\n" + e
)

# iterate and remove all the unwanted files from the git repo:
for item in [path.join(output_dir, i) for i in self._EXTRA_FILES]:
if not path.exists(item):
continue

if path.isdir(item) and not path.islink(item):
shutil.rmtree(item)
else:
os.remove(item)

# rename handlers.sh to <charm-name>.sh
new_name = '%s.sh' % config['metadata']['package'].replace('-', '_')
os.rename(os.path.join(output_dir, 'reactive', 'handlers.sh'),
os.path.join(output_dir, 'reactive', new_name))
19 changes: 19 additions & 0 deletions charmtools/templates/reactive_python/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/python
#
# Copyright (C) 2014 Canonical Ltd.
# Author: Clint Byrum <clint.byrum@canonical.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from .template import ReactivePythonCharmTemplate # noqa
98 changes: 98 additions & 0 deletions charmtools/templates/reactive_python/template.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/usr/bin/python
#
# Copyright (C) 2014 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import logging
import os
import os.path as path
import time
import shutil
import subprocess
import tempfile

from Cheetah.Template import Template
from stat import ST_MODE

from charmtools.generators import (
CharmTemplate,
)

log = logging.getLogger(__name__)


class ReactivePythonCharmTemplate(CharmTemplate):
"""Creates a reactive, layered python-based charm"""

# _EXTRA_FILES is the list of names of files present in the git repo
# we don't want transferred over to the charm template:
_EXTRA_FILES = ["README.md", ".git", ".gitmodules"]

_TEMPLATE_URL = "https://github.com/juju-solutions/template-reactive-python"

def create_charm(self, config, output_dir):
self._clone_template(config, output_dir)

for root, dirs, files in os.walk(output_dir):
for outfile in files:
if self.skip_template(outfile):
continue

self._template_file(config, path.join(root, outfile))

def _template_file(self, config, outfile):
if path.islink(outfile):
return

mode = os.stat(outfile)[ST_MODE]
t = Template(file=outfile, searchList=(config))
o = tempfile.NamedTemporaryFile(
dir=path.dirname(outfile), delete=False)
os.chmod(o.name, mode)
o.write(str(t))
o.close()
backupname = outfile + str(time.time())
os.rename(outfile, backupname)
os.rename(o.name, outfile)
os.unlink(backupname)

def _clone_template(self, config, output_dir):
cmd = "git clone --recursive {} {}".format(
self._TEMPLATE_URL, output_dir
)

try:
subprocess.check_call(cmd.split())
except OSError as e:
raise Exception(
"The below error has occurred whilst attempting to clone"
"the charm template. Please make sure you have git"
"installed on your system.\n" + e
)

# iterate and remove all the unwanted files from the git repo:
for item in [path.join(output_dir, i) for i in self._EXTRA_FILES]:
if not path.exists(item):
continue

if path.isdir(item) and not path.islink(item):
shutil.rmtree(item)
else:
os.remove(item)

# rename handlers.py to <charm-name>.py
new_name = '%s.py' % config['metadata']['package'].replace('-', '_')
os.rename(os.path.join(output_dir, 'reactive', 'handlers.py'),
os.path.join(output_dir, 'reactive', new_name))
2 changes: 1 addition & 1 deletion charmtools/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,7 @@ def setup_parser():
parser = argparse.ArgumentParser(
prog='juju test',
formatter_class=argparse.RawDescriptionHelpFormatter,
description='Execute a charms functional tests',
description='execute charm functional tests',
epilog="""\
`%(prog)s` should always be run from within a CHARM_DIR.
Expand Down
68 changes: 0 additions & 68 deletions charmtools/update.py

This file was deleted.

37 changes: 27 additions & 10 deletions charmtools/utils.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import argparse
import copy
import collections
import hashlib
Expand Down Expand Up @@ -52,13 +53,17 @@ def deepmerge(dest, src):
for k, v in src.iteritems():
if dest.get(k) and isinstance(v, dict):
deepmerge(dest[k], v)
elif dest.get(k) and isinstance(v, list):
if not v in dest.get(k):
dest[k].extend(v)
else:
dest[k] = copy.deepcopy(v)
return dest


def delete_path(path, obj):
"""Delete a dotted path from object, assuming each level is a dict"""
# TODO: Support lists
parts = path.split('.')
for p in parts[:-1]:
obj = obj[p]
Expand Down Expand Up @@ -147,26 +152,27 @@ def __bool__(self):

__nonzero__ = __bool__

def throw_on_error(self):
def exit_on_error(self):
if not bool(self):
raise subprocess.CalledProcessError(
self.exit_code, self.command, output=self.output)
sys.stderr.write(
'{}\n\nCommand failed: {}\n'.format(self.output, self.cmd))
sys.exit(self.exit_code)


class Process(object):
def __init__(self, command=None, throw=False, log=log, **kwargs):
def __init__(self, command=None, exit=False, log=log, **kwargs):
if isinstance(command, str):
command = (command, )
self.command = command
self._throw_on_error = False
self._exit_on_error = exit
self.log = log
self._kw = kwargs

def __repr__(self):
return "<Command %s>" % (self.command, )

def throw_on_error(self, throw=True):
self._throw_on_error = throw
def exit_on_error(self, exit=True):
self._exit_on_error = exit
return self

def __call__(self, *args, **kw):
Expand All @@ -192,8 +198,8 @@ def __call__(self, *args, **kw):
exit_code = p.poll()
result = ProcessResult(all_args, exit_code, stdout, stderr)
self.log.debug("process: %s (%d)", result.cmd, result.exit_code)
if self._throw_on_error:
result.throw_on_error()
if self._exit_on_error:
result.exit_on_error()
return result

command = Process
Expand All @@ -211,7 +217,7 @@ def __getattr__(self, key):

def check(self, *args, **kwargs):
kwargs.update({'log': self.log})
return command(command=args, **kwargs).throw_on_error()
return command(command=args, **kwargs).exit_on_error()

def __call__(self, *args, **kwargs):
kwargs.update({'log': self.log})
Expand Down Expand Up @@ -520,3 +526,14 @@ def prefix_lines(lines, lineno):
m=message)
i += 1
return i == 0


class Description(argparse._StoreTrueAction):
"""A argparse action that prints its parent parser's description and exits."""
def __call__(self, parser, namespace, values, option_string=None):
print(parser.description.split('\n')[0].strip('. '))
raise SystemExit()


def add_plugin_description(parser):
parser.add_argument('--description', action=Description)
36 changes: 35 additions & 1 deletion charmtools/version.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,44 @@

import pkg_resources
import argparse

from cli import parser_defaults
from charmtools import utils


def get_args(args=None):
parser = argparse.ArgumentParser(
description='display tooling version information')
utils.add_plugin_description(parser)
parser = parser_defaults(parser)
args = parser.parse_args(args)

return args


def charm_version():
try:
from apt.cache import Cache
charm_vers = Cache()['charm'].versions
for v in charm_vers:
if v.is_installed:
charm_ver = v.version
break
except ImportError:
charm_ver = 'unavailable'
except:
charm_ver = 'error'

return charm_ver


def main():
get_args()

version = pkg_resources.get_distribution("charm-tools").version
print "%s %s" % ('charm-tools', version)

print "charm %s" % charm_version()
print "charm-tools %s" % version


if __name__ == '__main__':
Expand Down
6 changes: 3 additions & 3 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
PyYAML==3.11
blessings==1.6
bzr>=2.6.0
charmworldlib>=0.4.2
libcharmstore>=0.0.3
cheetah==2.4.4
colander==1.0
coverage==3.7.1
Expand All @@ -15,10 +15,10 @@ mock==1.0.1
nose==1.2.1
oauth==1.0.1
otherstuf==1.1.0
path.py==7.4
path.py==8.1.2
pathspec==0.3.3
pip>=7.1.2
requests==2.7.0
requests==2.6.0
responses==0.4.0
ruamel.yaml==0.10.2
virtualenv>=1.11.4
Expand Down
Loading