Enable Bionic as a gate test

Change bionic test from dev to gate for 18.05.

Change-Id: I21fdefe0aa0b019d5b211bb54d0a2fa9e38c2864

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./unit_tests
+top_dir=./

actions/actions.py

@@ -17,9 +17,23 @@
 import sys
 import os
 
+_path = os.path.dirname(os.path.realpath(__file__))
+_parent = os.path.abspath(os.path.join(_path, ".."))
+_hooks = os.path.abspath(os.path.join(_parent, "hooks"))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+
+_add_path(_parent)
+_add_path(_hooks)
+
+
 from charmhelpers.core.hookenv import action_fail
 
-from hooks.glance_utils import (
+from glance_utils import (
     pause_unit_helper,
     resume_unit_helper,
     register_configs,

actions/openstack_upgrade.py

@@ -14,16 +14,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import sys
+
+_path = os.path.dirname(os.path.realpath(__file__))
+_parent = os.path.abspath(os.path.join(_path, ".."))
+_hooks = os.path.abspath(os.path.join(_parent, "hooks"))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+
+_add_path(_parent)
+_add_path(_hooks)
+
+
 from charmhelpers.contrib.openstack.utils import (
     do_action_openstack_upgrade,
 )
 
-from hooks.glance_relations import (
+from glance_relations import (
     config_changed,
     CONFIGS
 )
 
-from hooks.glance_utils import do_openstack_upgrade
+from glance_utils import do_openstack_upgrade
 
 
 def openstack_upgrade():

charmhelpers/contrib/openstack/context.py

@@ -797,9 +797,9 @@ def configure_cert(self, cn=None):
             key_filename = 'key'
 
         write_file(path=os.path.join(ssl_dir, cert_filename),
-                   content=b64decode(cert))
+                   content=b64decode(cert), perms=0o640)
         write_file(path=os.path.join(ssl_dir, key_filename),
-                   content=b64decode(key))
+                   content=b64decode(key), perms=0o640)
 
     def configure_ca(self):
         ca_cert = get_ca_cert()
@@ -1873,10 +1873,11 @@ class EnsureDirContext(OSContextGenerator):
     context is needed to do that before rendering a template.
    '''
 
-    def __init__(self, dirname):
+    def __init__(self, dirname, **kwargs):
         '''Used merely to ensure that a given directory exists.'''
         self.dirname = dirname
+        self.kwargs = kwargs
 
     def __call__(self):
-        mkdir(self.dirname)
+        mkdir(self.dirname, **self.kwargs)
         return {}

charmhelpers/contrib/openstack/templates/section-oslo-middleware

@@ -0,0 +1,5 @@
+[oslo_middleware]
+
+# Bug #1758675
+enable_proxy_headers_parsing = true
+

charmhelpers/contrib/openstack/templates/section-oslo-notifications

@@ -5,4 +5,7 @@ transport_url = {{ transport_url }}
 {% if notification_topics -%}
 topics = {{ notification_topics }}
 {% endif -%}
+{% if notification_format -%}
+notification_format = {{ notification_format }}
+{% endif -%}
 {% endif -%}

charmhelpers/contrib/openstack/utils.py

@@ -306,7 +306,7 @@ def get_os_codename_install_source(src):
 
     if src.startswith('cloud:'):
         ca_rel = src.split(':')[1]
-        ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
+        ca_rel = ca_rel.split('-')[1].split('/')[0]
         return ca_rel
 
     # Best guess match based on deb string provided

charmhelpers/contrib/openstack/vaultlocker.py

@@ -0,0 +1,126 @@
+# Copyright 2018 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+import charmhelpers.contrib.openstack.alternatives as alternatives
+import charmhelpers.contrib.openstack.context as context
+
+import charmhelpers.core.hookenv as hookenv
+import charmhelpers.core.host as host
+import charmhelpers.core.templating as templating
+import charmhelpers.core.unitdata as unitdata
+
+VAULTLOCKER_BACKEND = 'charm-vaultlocker'
+
+
+class VaultKVContext(context.OSContextGenerator):
+    """Vault KV context for interaction with vault-kv interfaces"""
+    interfaces = ['secrets-storage']
+
+    def __init__(self, secret_backend=None):
+        super(context.OSContextGenerator, self).__init__()
+        self.secret_backend = (
+            secret_backend or 'charm-{}'.format(hookenv.service_name())
+        )
+
+    def __call__(self):
+        db = unitdata.kv()
+        last_token = db.get('last-token')
+        secret_id = db.get('secret-id')
+        for relation_id in hookenv.relation_ids(self.interfaces[0]):
+            for unit in hookenv.related_units(relation_id):
+                data = hookenv.relation_get(unit=unit,
+                                            rid=relation_id)
+                vault_url = data.get('vault_url')
+                role_id = data.get('{}_role_id'.format(hookenv.local_unit()))
+                token = data.get('{}_token'.format(hookenv.local_unit()))
+
+                if all([vault_url, role_id, token]):
+                    token = json.loads(token)
+                    vault_url = json.loads(vault_url)
+
+                    # Tokens may change when secret_id's are being
+                    # reissued - if so use token to get new secret_id
+                    if token != last_token:
+                        secret_id = retrieve_secret_id(
+                            url=vault_url,
+                            token=token
+                        )
+                        db.set('secret-id', secret_id)
+                        db.set('last-token', token)
+                        db.flush()
+
+                    ctxt = {
+                        'vault_url': vault_url,
+                        'role_id': json.loads(role_id),
+                        'secret_id': secret_id,
+                        'secret_backend': self.secret_backend,
+                    }
+                    vault_ca = data.get('vault_ca')
+                    if vault_ca:
+                        ctxt['vault_ca'] = json.loads(vault_ca)
+                    self.complete = True
+                    return ctxt
+        return {}
+
+
+def write_vaultlocker_conf(context, priority=100):
+    """Write vaultlocker configuration to disk and install alternative
+
+    :param context: Dict of data from vault-kv relation
+    :ptype: context: dict
+    :param priority: Priority of alternative configuration
+    :ptype: priority: int"""
+    charm_vl_path = "/var/lib/charm/{}/vaultlocker.conf".format(
+        hookenv.service_name()
+    )
+    host.mkdir(os.path.dirname(charm_vl_path), perms=0o700)
+    templating.render(source='vaultlocker.conf.j2',
+                      target=charm_vl_path,
+                      context=context, perms=0o600),
+    alternatives.install_alternative('vaultlocker.conf',
+                                     '/etc/vaultlocker/vaultlocker.conf',
+                                     charm_vl_path, priority)
+
+
+def vault_relation_complete(backend=None):
+    """Determine whether vault relation is complete
+
+    :param backend: Name of secrets backend requested
+    :ptype backend: string
+    :returns: whether the relation to vault is complete
+    :rtype: bool"""
+    vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND)
+    vault_kv()
+    return vault_kv.complete
+
+
+# TODO: contrib a high level unwrap method to hvac that works
+def retrieve_secret_id(url, token):
+    """Retrieve a response-wrapped secret_id from Vault
+
+    :param url: URL to Vault Server
+    :ptype url: str
+    :param token: One shot Token to use
+    :ptype token: str
+    :returns: secret_id to use for Vault Access
+    :rtype: str"""
+    import hvac
+    client = hvac.Client(url=url, token=token)
+    response = client._post('/v1/sys/wrapping/unwrap')
+    if response.status_code == 200:
+        data = response.json()
+        return data['data']['secret_id']

charmhelpers/contrib/storage/linux/ceph.py

@@ -291,7 +291,7 @@ def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT):
 
 class ReplicatedPool(Pool):
     def __init__(self, service, name, pg_num=None, replicas=2,
-                 percent_data=10.0):
+                 percent_data=10.0, app_name=None):
         super(ReplicatedPool, self).__init__(service=service, name=name)
         self.replicas = replicas
         if pg_num:
@@ -301,6 +301,10 @@ def __init__(self, service, name, pg_num=None, replicas=2,
             self.pg_num = min(pg_num, max_pgs)
         else:
             self.pg_num = self.get_pgs(self.replicas, percent_data)
+        if app_name:
+            self.app_name = app_name
+        else:
+            self.app_name = 'unknown'
 
     def create(self):
         if not pool_exists(self.service, self.name):
@@ -313,17 +317,27 @@ def create(self):
                 update_pool(client=self.service,
                             pool=self.name,
                             settings={'size': str(self.replicas)})
+                try:
+                    set_app_name_for_pool(client=self.service,
+                                          pool=self.name,
+                                          name=self.app_name)
+                except CalledProcessError:
+                    log('Could not set app name for pool {}'.format(self.name, level=WARNING))
             except CalledProcessError:
                 raise
 
 
 # Default jerasure erasure coded pool
 class ErasurePool(Pool):
     def __init__(self, service, name, erasure_code_profile="default",
-                 percent_data=10.0):
+                 percent_data=10.0, app_name=None):
         super(ErasurePool, self).__init__(service=service, name=name)
         self.erasure_code_profile = erasure_code_profile
         self.percent_data = percent_data
+        if app_name:
+            self.app_name = app_name
+        else:
+            self.app_name = 'unknown'
 
     def create(self):
         if not pool_exists(self.service, self.name):
@@ -355,6 +369,12 @@ def create(self):
                    'erasure', self.erasure_code_profile]
             try:
                 check_call(cmd)
+                try:
+                    set_app_name_for_pool(client=self.service,
+                                          pool=self.name,
+                                          name=self.app_name)
+                except CalledProcessError:
+                    log('Could not set app name for pool {}'.format(self.name, level=WARNING))
             except CalledProcessError:
                 raise
 
@@ -778,6 +798,25 @@ def update_pool(client, pool, settings):
     check_call(cmd)
 
 
+def set_app_name_for_pool(client, pool, name):
+    """
+    Calls `osd pool application enable` for the specified pool name
+
+    :param client: Name of the ceph client to use
+    :type client: str
+    :param pool: Pool to set app name for
+    :type pool: str
+    :param name: app name for the specified pool
+    :type name: str
+
+    :raises: CalledProcessError if ceph call fails
+    """
+    if ceph_version() >= '12.0.0':
+        cmd = ['ceph', '--id', client, 'osd', 'pool',
+               'application', 'enable', pool, name]
+        check_call(cmd)
+
+
 def create_pool(service, name, replicas=3, pg_num=None):
     """Create a new RADOS pool."""
     if pool_exists(service, name):