utils.py

# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
from io import BytesIO
import json
from muranopkgcheck import manager as check_manager
from muranopkgcheck import pkg_loader as check_pkg_loader
from muranopkgcheck import validators as check_validators
import os
import re
import shutil
import sys
import tempfile
import textwrap
import uuid
import warnings
import zipfile

from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import encodeutils
from oslo_utils import uuidutils

import prettytable
import requests
import urllib
import yaml
import yaql

from muranoclient.common import exceptions
from muranoclient.i18n import _

try:
    import yaql.language  # noqa

    from muranoclient.common.yaqlexpression import YaqlExpression
except ImportError:
    # no yaql.language means legacy yaql
    from muranoclient.common.yaqlexpression_legacy import YaqlExpression


LOG = logging.getLogger(__name__)


# Decorator for cli-args
def arg(*args, **kwargs):
    def _decorator(func):
        # Because of the semantics of decorator composition if we just append
        # to the options list positional options will appear to be backwards.
        func.__dict__.setdefault('arguments', []).insert(0, (args, kwargs))
        return func
    return _decorator


def json_formatter(js):
    return jsonutils.dumps(js, indent=2)


def text_wrap_formatter(d):
    return '\n'.join(textwrap.wrap(d or '', 55))


def pretty_choice_list(l):
    return ', '.join("'%s'" % i for i in l)


def print_list(objs, fields, field_labels, formatters=None, sortby=0):
    if formatters is None:
        formatters = {}
    pt = prettytable.PrettyTable([f for f in field_labels], caching=False)
    pt.align = 'l'

    for o in objs:
        row = []
        for field in fields:
            if field in formatters:
                row.append(formatters[field](o))
            else:
                data = getattr(o, field, None) or ''
                row.append(data)
        pt.add_row(row)

    result = encodeutils.safe_encode(pt.get_string())

    result = result.decode()

    print(result)


def print_dict(d, formatters=None):
    if formatters is None:
        formatters = {}
    pt = prettytable.PrettyTable(['Property', 'Value'], caching=False)
    pt.align = 'l'

    for field in d.keys():
        if field in formatters:
            pt.add_row([field, formatters[field](d[field])])
        else:
            pt.add_row([field, d[field]])

    result = encodeutils.safe_encode(pt.get_string(sortby='Property'))

    result = result.decode()

    print(result)


def find_resource(manager, name_or_id, *args, **kwargs):
    """Helper for the _find_* methods."""
    # first try to get entity as integer id
    try:
        if isinstance(name_or_id, int) or name_or_id.isdigit():
            return manager.get(int(name_or_id), *args, **kwargs)
    except exceptions.NotFound:
        pass

    # now try to get entity as uuid
    try:
        uuid.UUID(str(name_or_id))
        return manager.get(name_or_id, *args, **kwargs)
    except (ValueError, exceptions.NotFound):
        pass

    # finally try to find entity by name
    try:
        return manager.find(name=name_or_id)
    except exceptions.NotFound:
        msg = "No %s with a name or ID of '%s' exists." % \
              (manager.resource_class.__name__.lower(), name_or_id)
        raise exceptions.CommandError(msg)


def string_to_bool(arg):
    return arg.strip().lower() in ('t', 'true', 'yes', '1')


def env(*vars, **kwargs):
    """Search for the first defined of possibly many env vars

    Returns the first environment variable defined in vars, or
    returns the default defined in kwargs.
    """
    for v in vars:
        value = os.environ.get(v, None)
        if value:
            return value
    return kwargs.get('default', '')


def exit(msg=''):
    if msg:
        print(encodeutils.safe_encode(msg), file=sys.stderr)
    sys.exit(1)


def getsockopt(self, *args, **kwargs):
    """Allows us to monkey patch eventlet's GreenSocket

    A function which allows us to monkey patch eventlet's
    GreenSocket, adding a required 'getsockopt' method.
    TODO: (mclaren) we can remove this once the eventlet fix
    (https://bitbucket.org/eventlet/eventlet/commits/609f230)
    lands in mainstream packages.
    NOTE: Already in 0.13, but we can't be sure that all clients
    that use python-muranoclient also use newest eventlet
    """
    return self.fd.getsockopt(*args, **kwargs)


def exception_to_str(exc):
    try:
        error = str(exc)
    except UnicodeError:
        error = ("Caught '%(exception)s' exception." %
                 {"exception": exc.__class__.__name__})
    return encodeutils.safe_encode(error, errors='ignore')


class NoCloseProxy(object):
    """A proxy object, that does nothing on close."""
    def __init__(self, obj):
        self.obj = obj

    def close(self):
        return

    def __getattr__(self, name):
        return getattr(self.obj, name)


class File(object):
    def __init__(self, name, binary=True):
        self.name = name
        self.binary = binary

    def open(self):
        mode = 'rb' if self.binary else 'r'
        if hasattr(self.name, 'read'):
            # NOTE(kzaitsev) We do not want to close a file object
            # passed to File wrapper. The caller should be responsible
            # for closing it
            return NoCloseProxy(self.name)
        else:
            if os.path.isfile(self.name):
                return open(self.name, mode)
            if os.path.isdir(self.name):
                tmp = tempfile.NamedTemporaryFile()
                archive = zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED)
                for root, dirs, files in os.walk(self.name):
                    for _file in files:
                        destination = os.path.relpath(
                            os.path.join(root, _file), os.path.join(self.name))
                        archive.write(os.path.join(root, _file), destination)
                tmp.flush()
                return open(tmp.name, mode)
            url = urllib.parse.urlparse(self.name)
            if url.scheme in ('http', 'https'):
                resp = requests.get(self.name, stream=True)
                if not resp.ok:
                    raise ValueError("Got non-ok status({0}) "
                                     "while connecting to {1}".format(
                                         resp.status_code, self.name))
                temp_file = tempfile.NamedTemporaryFile(mode='w+b')
                for chunk in resp.iter_content(1024 * 1024):
                    temp_file.write(chunk)
                temp_file.flush()
                return open(temp_file.name, mode)
            raise ValueError("Can't open {0}".format(self.name))


def to_url(filename, base_url, version='', path='/', extension=''):
    if urllib.parse.urlparse(filename).scheme in ('http', 'https'):
        return filename
    if not base_url:
        raise ValueError("No base_url for repository supplied")
    if '/' in filename or filename in ('.', '..'):
        raise ValueError("Invalid filename path supplied: {0}".format(
            filename))
    version = '.' + version if version else ''
    return urllib.parse.urljoin(base_url,
                                path + filename + version + extension)


class FileWrapperMixin(object):
    def __init__(self, file_wrapper):
        self.file_wrapper = file_wrapper
        try:
            self._file = self.file_wrapper.open()
        except Exception:
            # NOTE(kzaitsev): We need to have _file available at __del__ time.
            self._file = None
            raise

    def file(self):
        self._file.seek(0)
        return self._file

    def close(self):
        if self._file and not self._file.closed:
            self._file.close()

    def save(self, dst, binary=True):
        file_name = self.file_wrapper.name

        if urllib.parse.urlparse(file_name).scheme:
            file_name = file_name.split('/')[-1]

        dst = os.path.join(dst, file_name)

        mode = 'wb' if binary else 'w'
        with open(dst, mode) as dst_file:
            self._file.seek(0)
            shutil.copyfileobj(self._file, dst_file)

    def __del__(self):
        self.close()


class Package(FileWrapperMixin):
    """Represents murano package contents."""

    @staticmethod
    def from_file(file_obj):
        if not isinstance(file_obj, File):
            file_obj = File(file_obj)
        pkg = Package(file_obj)
        errs = pkg.validate()
        if errs:
            raise exceptions.HTTPBadRequest(details=errs)
        return pkg

    @staticmethod
    def fromFile(file_obj):
        warnings.warn("Use from_file function", DeprecationWarning)
        return Package.from_file(file_obj)

    @staticmethod
    def from_location(name, base_url='', version='', url='', path=None):
        """Open file using one of three possible options

        If path is supplied search for name file in the path, otherwise
        if url is supplied - open that url and finally search murano
        repository for the package.
        """
        if path:
            pkg_name = os.path.join(path, name)
            file_name = None
            for f in [pkg_name, pkg_name + '.zip']:
                if os.path.exists(f):
                    file_name = f
            if file_name:
                return Package.from_file(file_name)
            LOG.error("Couldn't find file for package {0}, tried {1}".format(
                name, [pkg_name, pkg_name + '.zip']))
        if url:
            return Package.from_file(url)
        return Package.from_file(to_url(
            name,
            base_url=base_url,
            version=version,
            path='apps/',
            extension='.zip')
        )

    def validate(self):
        m = check_manager.Manager(self._file,
                                  loader=check_pkg_loader.ZipLoader)
        errors = m.validate(
            validators=[check_validators.manifest.ManifestValidator],
            only_errors=True)
        if errors:
            fmt = check_manager.PlainTextFormatter().format
            return 'Invalid Murano package\n{}\n'.format(fmt(errors))

    @property
    def contents(self):
        """Contents of a package."""
        if not hasattr(self, '_contents'):
            try:
                self._file.seek(0)
                self._zip_obj = zipfile.ZipFile(
                    BytesIO(self._file.read()))
            except Exception as e:
                LOG.error("Error {0} occurred,"
                          " while parsing the package".format(e))
                raise
        return self._zip_obj

    @property
    def manifest(self):
        """Parsed manifest file of a package."""
        if not hasattr(self, '_manifest'):
            try:
                self._manifest = yaml.safe_load(
                    self.contents.open('manifest.yaml'))
            except Exception as e:
                LOG.error("Error {0} occurred, while extracting "
                          "manifest from package".format(e))
                raise
        return self._manifest

    def images(self):
        """Returns a list of required image specifications."""
        if 'images.lst' not in self.contents.namelist():
            return []
        try:
            return yaml.safe_load(
                self.contents.open('images.lst')).get('Images', [])
        except Exception:
            return []

    @property
    def resolvers(self):
        if not hasattr(self, '_resolvers'):
            self.classes
        return self._resolvers

    @property
    def classes(self):
        if not hasattr(self, '_classes'):
            self._classes = {}
            self._resolvers = {}
            for class_name, class_file in (
                    self.manifest.get('Classes', {}).items()):
                filename = "Classes/%s" % class_file
                if filename not in self.contents.namelist():
                    continue
                klass_list = yaml.load_all(self.contents.open(filename),
                                           DummyYaqlYamlLoader)
                if not klass_list:
                    raise ValueError('No classes defined in file')
                resolver = None
                for klass in klass_list:
                    ns = klass.get('Namespaces')
                    if ns:
                        resolver = NamespaceResolver(ns)
                    name = klass.get('Name')
                    if name and resolver:
                        name = resolver.resolve_name(name)
                    if name == class_name:
                        self._classes[class_name] = klass
                        self._resolvers[class_name] = resolver
                        break
        return self._classes

    @property
    def ui(self):
        if not hasattr(self, '_ui'):
            if 'UI/ui.yaml' in self.contents.namelist():
                self._ui = self.contents.open('UI/ui.yaml')
            else:
                self._ui = None
        return self._ui

    @property
    def logo(self):
        if not hasattr(self, '_logo'):
            if 'logo.png' in self.contents.namelist():
                self._logo = self.contents.open('logo.png')
            else:
                self._logo = None
        return self._logo

    def _get_package_order(self, packages_graph):
        """Sorts packages according to dependencies between them

        Murano allows cyclic dependencies. It is impossible
        to do topological sort for graph with cycles, so at first
        graph condensation should be built.
        For condensation building Kosaraju's algorithm is used.
        Packages in strongly connected components can be situated
        in random order to each other.
        """
        def topological_sort(graph, start_node):
            order = []
            not_seen = set(graph)

            def dfs(node):
                not_seen.discard(node)
                for dep_node in graph[node]:
                    if dep_node in not_seen:
                        dfs(dep_node)
                order.append(node)

            dfs(start_node)
            return order

        def transpose_graph(graph):
            transposed = collections.defaultdict(list)
            for node, deps in graph.items():
                for dep in deps:
                    transposed[dep].append(node)
            return transposed

        order = topological_sort(packages_graph, self.manifest['FullName'])
        order.reverse()
        transposed = transpose_graph(packages_graph)

        def top_sort_by_components(graph, component_order):
            result = []
            seen = set()

            def dfs(node):
                seen.add(node)
                result.append(node)
                for dep_node in graph[node]:
                    if dep_node not in seen:
                        dfs(dep_node)
            for item in component_order:
                if item not in seen:
                    dfs(item)
            return reversed(result)
        return top_sort_by_components(transposed, order)

    def requirements(self, base_url, path=None, dep_dict=None):
        """Scans Require section of manifests of all the dependencies.

        Returns a dict with FQPNs as keys and respective Package objects
        as values, ordered by topological sort.

        :param base_url: url of packages location
        :param path: local path of packages location
        :param dep_dict: unused. Left for backward compatibility
        """

        unordered_requirements = {}
        requirements_graph = collections.defaultdict(list)
        dep_queue = collections.deque([(self.manifest['FullName'], self)])
        while dep_queue:
            dep_name, dep_file = dep_queue.popleft()
            unordered_requirements[dep_name] = dep_file
            direct_deps = Package._get_direct_deps(dep_file, base_url, path)
            for name, file in direct_deps:
                if name not in unordered_requirements:
                    dep_queue.append((name, file))
            requirements_graph[dep_name] = [dep[0] for dep in direct_deps]

        ordered_reqs_names = self._get_package_order(requirements_graph)
        ordered_reqs_dict = collections.OrderedDict()
        for name in ordered_reqs_names:
            ordered_reqs_dict[name] = unordered_requirements[name]

        return ordered_reqs_dict

    @staticmethod
    def _get_direct_deps(package, base_url, path):
        result = []
        if 'Require' in package.manifest:
            for dep_name, ver in package.manifest['Require'].items():
                try:
                    req_file = Package.from_location(
                        dep_name,
                        version=ver,
                        path=path,
                        base_url=base_url,
                    )
                except Exception as e:
                    LOG.error("Error {0} occurred while parsing package {1}, "
                              "required by {2} package".format(
                                  e, dep_name,
                                  package.manifest['FullName']))
                    continue
                result.append((req_file.manifest['FullName'], req_file))
        return result


def save_image_local(image_spec, base_url, dst):
    dst = os.path.join(dst, image_spec['Name'])

    download_url = to_url(
        image_spec.get("Url", image_spec['Name']),
        base_url=base_url,
        path='images/'
    )

    with open(dst, "w") as image_file:
        response = requests.get(download_url, stream=True)
        total_length = response.headers.get('content-length')

        if total_length is None:
            image_file.write(response.content)
        else:
            dl = 0
            total_length = int(total_length)
            for chunk in response.iter_content(1024 * 1024):
                dl += len(chunk)
                image_file.write(chunk)
                done = int(50 * dl / total_length)
                sys.stdout.write("\r[{0}{1}]".
                                 format('=' * done, ' ' * (50 - done)))
                sys.stdout.flush()
            sys.stdout.write("\n")
            image_file.flush()


def ensure_images(glance_client, image_specs, base_url,
                  local_path=None,
                  is_package_public=False):
    """Ensure that images are available

    Ensure that images from image_specs are available in glance. If not
    attempts: instructs glance to download the images and sets murano-specific
    metadata for it.
    """
    def _image_valid(image, keys):
        for key in keys:
            if key not in image:
                LOG.warning("Image specification invalid: "
                            "No {0} key in image ").format(key)
                return False
        return True

    keys = ['Name', 'DiskFormat', 'ContainerFormat', ]
    installed_images = []
    for image_spec in image_specs:
        if not _image_valid(image_spec, keys):
            continue
        filters = {
            'name': image_spec["Name"],
            'disk_format': image_spec["DiskFormat"],
            'container_format': image_spec["ContainerFormat"],
        }

        images = glance_client.images.list(filters=filters)
        try:
            img = next(images).to_dict()
        except StopIteration:
            img = None

        update_metadata = False
        if img:
            LOG.info("Found desired image {0}, id {1}".format(
                img['name'], img['id']))
            # check for murano meta-data
            if 'murano_image_info' in img.get('properties', {}):
                LOG.info("Image {0} already has murano meta-data".format(
                    image_spec['Name']))
            else:
                update_metadata = True
        else:
            LOG.info("Desired image {0} not found attempting "
                     "to download".format(image_spec['Name']))
            update_metadata = True

            img_file = None
            if local_path:
                img_file = os.path.join(local_path, image_spec['Name'])

            if img_file and not os.path.exists(img_file):
                LOG.error("Image file {0} does not exist."
                          .format(img_file))

            if img_file and os.path.exists(img_file):
                img = glance_client.images.create(
                    name=image_spec['Name'],
                    container_format=image_spec['ContainerFormat'],
                    disk_format=image_spec['DiskFormat'],
                    data=open(img_file, 'rb'),
                )
                img = img.to_dict()
            else:
                download_url = to_url(
                    image_spec.get("Url", image_spec['Name']),
                    base_url=base_url,
                    path='images/',
                )
                LOG.info("Instructing glance to download image {0}".format(
                    image_spec['Name']))
                img = glance_client.images.create(
                    name=image_spec["Name"],
                    container_format=image_spec['ContainerFormat'],
                    disk_format=image_spec['DiskFormat'],
                    copy_from=download_url)
                img = img.to_dict()

            if is_package_public:
                try:
                    glance_client.images.update(img['id'], is_public=True)
                    LOG.debug('Success update for image {0}'.format(img['id']))
                except Exception as e:
                    LOG.exception(_("Error {0} occurred while setting "
                                    "image {1} public").format(e, img['id']))

            installed_images.append(img)

            if update_metadata and 'Meta' in image_spec:
                LOG.info("Updating image {0} metadata".format(
                    image_spec['Name']))
                murano_image_info = jsonutils.dumps(image_spec['Meta'])
                glance_client.images.update(
                    img['id'], properties={'murano_image_info':
                                           murano_image_info})
    return installed_images


class Bundle(FileWrapperMixin):
    """Represents murano bundle contents."""

    @staticmethod
    def from_file(file_obj):
        if not isinstance(file_obj, File):
            file_obj = File(file_obj, binary=False)
        return Bundle(file_obj)

    @staticmethod
    def fromFile(file_obj):
        warnings.warn("Use from_file function", DeprecationWarning)
        return Bundle.from_file(file_obj)

    def package_specs(self):
        """Get a generator yielding package specifications

        Returns a generator yielding package specifications i.e.
        dicts with 'Name' and 'Version' fields
        """
        self._file.seek(0)
        bundle = None
        try:
            # NOTE(kzaitsev) jsonutils throws a type error here
            # see bug 1515231
            bundle = json.load(self._file)
        except ValueError:
            pass
        if bundle is None:
            try:
                bundle = yaml.safe_load(self._file)
            except yaml.error.YAMLError:
                pass

        if bundle is None or 'Packages' not in bundle:
            raise ValueError("Can't parse bundle contents")

        for package in bundle['Packages']:
            if 'Name' not in package:
                continue
            yield package

    def packages(self, base_url='', path=None):
        """Get a generator yielding Package objects

        Returns a generator, yielding Package objects for each package
        found in the bundle.
        """
        for package in self.package_specs():
            try:
                pkg_obj = Package.from_location(
                    package['Name'],
                    version=package.get('Version'),
                    url=package.get('Url'),
                    path=path,
                    base_url=base_url,
                )

            except Exception as e:
                LOG.error("Error {0} occurred while obtaining "
                          "package {1}".format(e, package['Name']))
                continue
            yield pkg_obj


class DummyYaqlYamlLoader(yaml.SafeLoader):
    """Constructor that treats !yaql as string."""
    pass


DummyYaqlYamlLoader.add_constructor(
    u'!yaql', DummyYaqlYamlLoader.yaml_constructors[u'tag:yaml.org,2002:str'])


class YaqlYamlLoader(yaml.SafeLoader):
    pass


# workaround for PyYAML bug: http://pyyaml.org/ticket/221
resolvers = {}
for k, v in yaml.SafeLoader.yaml_implicit_resolvers.items():
    resolvers[k] = v[:]
YaqlYamlLoader.yaml_implicit_resolvers = resolvers


def yaql_constructor(loader, node):
    value = loader.construct_scalar(node)
    return YaqlExpression(value)


YaqlYamlLoader.add_constructor(u'!yaql', yaql_constructor)
YaqlYamlLoader.add_implicit_resolver(u'!yaql', YaqlExpression, None)


def traverse_and_replace(obj,
                         pattern=re.compile(r'^===id(\d+)===$'),
                         replacements=None):
    """Helper function that traverses object model and substitutes ids.

    Recursively checks values of objects found in `obj` against `pattern`,
    and replaces strings that match pattern with uuidutils.generate_uuid().
    Keeps track of any replacements already made, i.e. ===id1=== would
    always be the same, across `obj`. Uses 1st group, found in the `pattern`
    regexp as unique identifier of a replacement
    """
    if replacements is None:
        replacements = collections.defaultdict(
            lambda: uuidutils.generate_uuid(dashed=False))

    def _maybe_replace(obj, key, value):
        """Check and replace value against pattern"""
        if isinstance(value, str):
            m = pattern.search(value)
            if m:
                if m.group(1) not in replacements:
                    replacements[m.group(1)] = uuidutils.generate_uuid(
                        dashed=False)
                obj[key] = replacements[m.group(1)]

    if isinstance(obj, list):
        for key, value in enumerate(obj):
            if isinstance(value, (list, dict)):
                traverse_and_replace(value, pattern, replacements)
            else:
                _maybe_replace(obj, key, value)
    elif isinstance(obj, dict):
        for key, value in obj.items():
            if isinstance(value, (list, dict)):
                traverse_and_replace(value, pattern, replacements)
            else:
                _maybe_replace(obj, key, value)
    else:
        _maybe_replace(obj, key, value)


class NamespaceResolver(object):
    """Copied from main murano repo

    original at murano/dsl/namespace_resolver.py
    """

    def __init__(self, namespaces):
        self._namespaces = namespaces
        self._namespaces[''] = ''

    def resolve_name(self, name, relative=None):
        if name is None:
            raise ValueError()
        if name and name.startswith(':'):
            return name[1:]
        if ':' in name:
            parts = name.split(':')
            if len(parts) != 2 or not parts[1]:
                raise NameError('Incorrectly formatted name ' + name)
            if parts[0] not in self._namespaces:
                raise KeyError('Unknown namespace prefix ' + parts[0])
            return '.'.join((self._namespaces[parts[0]], parts[1]))
        if not relative and '=' in self._namespaces and '.' not in name:
            return '.'.join((self._namespaces['='], name))
        if relative and '.' not in name:
            return '.'.join((relative, name))
        return name