diff --git a/.gitignore b/.gitignore index 31762270..02d5df5e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ local django-cloudlaunch/cloudlaunchserver/settings_local.py django-cloudlaunch/db.sqlite3* -django-cloudlaunch/cloudlaunch.log +django-cloudlaunch/cloudlaunch*.log django-cloudlaunch/cloudlaunchserver/db.sqlite3 django-cloudlaunch/baselaunch/backend_plugins/rancher_ansible* diff --git a/django-cloudlaunch/cloudlaunch/admin.py b/django-cloudlaunch/cloudlaunch/admin.py index 02621b34..38a1c186 100644 --- a/django-cloudlaunch/cloudlaunch/admin.py +++ b/django-cloudlaunch/cloudlaunch/admin.py @@ -71,8 +71,18 @@ def instance_type(self, obj): return app_config.get('config_cloudlaunch', {}).get('instanceType') +class PublicKeyInline(admin.StackedInline): + model = models.PublicKey + extra = 1 + + +class UserProfileAdmin(admin.ModelAdmin): + inlines = [PublicKeyInline] + + admin.site.register(models.Application, AppAdmin) admin.site.register(models.AppCategory, AppCategoryAdmin) admin.site.register(models.ApplicationDeployment, AppDeploymentsAdmin) admin.site.register(models.CloudImage, CloudImageAdmin) admin.site.register(models.Usage, UsageAdmin) +admin.site.register(models.UserProfile, UserProfileAdmin) diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/app_plugin.py b/django-cloudlaunch/cloudlaunch/backend_plugins/app_plugin.py index bec7962e..278ba803 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/app_plugin.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/app_plugin.py @@ -61,43 +61,101 @@ def sanitise_app_config(app_config): pass @abc.abstractmethod - def launch_app(self, provider, task, name, cloud_config, app_config, - user_data): + def deploy(self, name, task, app_config, provider_config): """ - Launch a given application on the target infrastructure. + Deploy this app plugin on the supplied provider. + + Perform all the necessary steps to deploy this appliance. This may + involve provisioning cloud resources or configuring existing host(s). + See the definition of each method argument as some have required + structure. This operation is designed to be a Celery task, and thus, can contain long-running operations. - @type provider: :class:`CloudBridge.CloudProvider` - @param provider: Cloud provider where the supplied deployment is to be - created. + @type name: ``str`` + @param name: Name of this deployment. @type task: :class:`Task` @param task: A Task object, which can be used to report progress. See ``tasks.Task`` for the interface details and sample implementation. - @type name: ``str`` - @param name: Name of this deployment. - - @type cloud_config: ``dict`` - @param cloud_config: A dict containing cloud infrastructure specific - configuration for this app. - @type app_config: ``dict`` - @param app_config: A dict containing the original, unprocessed version - of the app config. The app config is a merged dict - of database stored settings and user-entered - settings. - - @type user_data: ``object`` - @param user_data: An object returned by the ``process_app_config()`` - method which contains a validated and processed - version of the ``app_config``. + @param app_config: A dict containing the appliance configuration. The + app config is a merged dict of database stored + settings and user-entered settings. In addition to + the static configuration of the app, such as + firewall rules or access password, this should + contain a url to a host configuration playbook, if + such configuration step is desired. For example: + ``` +{ + "config_cloudman": {}, + "config_appliance": { + "sshUser": "ubuntu", + "runner": "ansible", + "repository": "https://github.com/afgane/Rancher-Ansible", + "inventoryTemplate": "https://gist.githubusercontent.com/..." + }, + "config_cloudlaunch": { + "vmType": "c3.large", + "firewall": [ { + "securityGroup": "cloudlaunch-cm2", + "rules": [ { + "protocol": "tcp", + "from": "22", + "to": "22", + "cidr": "0.0.0.0/0" + } ] } ] } } +``` + @type provider_config: ``dict`` + @param provider_config: Define the details of of the infrastructure + provider where the appliance should be + deployed. It is expected that this dictionary + is composed within a task calling the plugin so + it reflects the supplied info and derived + properties. See ``tasks.py → create_appliance`` + for an example. + The following keys are supported: + * ``cloud_provider``: CloudBridge object of the + cloud provider + * ``cloud_config``: A dict containing cloud + infrastructure specific + configuration for this app + * ``cloud_user_data``: An object returned by + ``process_app_config()`` + method which contains a + validated and formatted + version of the + ``app_config`` to be + supplied as instance + user data + * ``host_address``: A host IP address or a + hostnames where to deploy + this appliance + * ``ssh_user``: User name with which to access + the host(s) + * ``ssh_public_key``: Public RSA ssh key to be + used when running the app + configuration step. This + should be the actual key. + CloudLaunch will auto-gen + this key for provisioned + instances. For hosted + instances, the user + should retrieve + CloudLaunch's public key + but this value should not + be supplied. + * ``ssh_private_key``: Private portion of an + RSA ssh key. This should + not be supplied by a + user and is intended + only for internal use. :rtype: ``dict`` - :return: Results of the launch process. + :return: Results of the deployment process. """ pass diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py index 855dc495..a6cfc10a 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py @@ -4,15 +4,14 @@ import yaml import ipaddress +from celery.utils.log import get_task_logger +from cloudbridge.cloud.base.helpers import generate_key_pair from cloudbridge.cloud.interfaces import InstanceState from cloudbridge.cloud.interfaces.resources import TrafficDirection -import requests -import requests.exceptions from .app_plugin import AppPlugin -import logging -log = logging.getLogger(__name__) +log = get_task_logger('cloudlaunch') class BaseVMAppPlugin(AppPlugin): @@ -247,10 +246,39 @@ def resolve_launch_properties(self, provider, cloudlaunch_config): provider, subnet, cloudlaunch_config['firewall']) return subnet, placement, vmf - def launch_app(self, provider, task, name, cloud_config, - app_config, user_data): - """Initiate the app launch process.""" + def deploy(self, name, task, app_config, provider_config): + """See the parent class in ``app_plugin.py`` for the docstring.""" + p_result = {} + c_result = {} + if provider_config.get('host_address'): + # A host is provided; use CloudLaunch's default published ssh key + pass # Implement this once we actually support it + else: + if app_config.get('config_appliance'): + # Host config will take place; generate a tmp ssh config key + public_key, private_key = generate_key_pair() + provider_config['ssh_private_key'] = private_key + provider_config['ssh_public_key'] = public_key + provider_config['ssh_user'] = app_config.get( + 'config_appliance', {}).get('sshUser') + p_result = self._provision_host(name, task, app_config, + provider_config) + provider_config['host_address'] = p_result['cloudLaunch'].get( + 'publicIP') + + if app_config.get('config_appliance'): + c_result = self._configure_host(name, task, app_config, + provider_config) + # Merge result dicts; right-most dict keys take precedence + return {**p_result, **c_result} + + def _provision_host(self, name, task, app_config, provider_config): + """Provision a host using the provider_config info.""" cloudlaunch_config = app_config.get("config_cloudlaunch", {}) + provider = provider_config.get('cloud_provider') + cloud_config = provider_config.get('cloud_config') + user_data = provider_config.get('cloud_user_data') or "" + custom_image_id = cloudlaunch_config.get("customImageID", None) img = provider.compute.images.get( custom_image_id or cloud_config.get('image_id')) @@ -268,27 +296,35 @@ def launch_app(self, provider, task, name, cloud_config, vm_type = cloudlaunch_config.get( 'vmType', cloud_config.get('default_instance_type')) - log.debug("Launching with subnet %s and VM firewalls %s" % - (subnet, vmfl)) - log.info("Launching base_vm of type %s with UD:\n%s" % (vm_type, - user_data)) - task.update_state(state='PROGRESSING', - meta={'action': "Launching an instance of type %s " - "with keypair %s in zone %s" % - (vm_type, kp.name, placement_zone)}) + log.debug("Launching with subnet %s and VM firewalls %s", subnet, vmfl) + + if provider_config.get('ssh_public_key'): + # cloud-init config to allow login w/ the config ssh key + # http://cloudinit.readthedocs.io/en/latest/topics/examples.html + log.info("Adding a cloud-init config public ssh key to user data") + user_data += """ +#cloud-config +ssh_authorized_keys: + - {0}""".format(provider_config['ssh_public_key']) + log.info("Launching base_vm of type %s with UD:\n%s", vm_type, + user_data) + task.update_state(state="PROGRESSING", + meta={"action": "Launching an instance of type %s " + "with keypair %s in zone %s" % + (vm_type, kp.name, placement_zone)}) inst = provider.compute.instances.create( name=name, image=img, vm_type=vm_type, subnet=subnet, key_pair=kp, vm_firewalls=vmfl, zone=placement_zone, user_data=user_data, launch_config=cb_launch_config) - task.update_state(state='PROGRESSING', - meta={'action': "Waiting for instance %s" % inst.id}) + task.update_state(state="PROGRESSING", + meta={"action": "Waiting for instance %s" % inst.id}) log.debug("Waiting for instance {0} to be ready...".format(inst.id)) inst.wait_till_ready() static_ip = cloudlaunch_config.get('staticIP') if static_ip: task.update_state(state='PROGRESSING', meta={'action': "Assigning requested floating " - "IP: %s" % static_ip}) + "IP: %s" % static_ip}) inst.add_floating_ip(static_ip) inst.refresh() results = {} @@ -306,6 +342,14 @@ def launch_app(self, provider, task, name, cloud_config, "Public IP: %s" % results.get('publicIP') or ""}) return {"cloudLaunch": results} + def _configure_host(self, name, task, app_config, provider_config): + host = provider_config.get('host_address') + task.update_state( + state='PROGRESSING', + meta={"action": "Configuring host % s" % host}) + log.info("Configuring host %s", host) + return {} + def _get_deployment_iid(self, deployment): """ Extract instance ID for the supplied deployment. diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py index 6e1e0b8f..35e8a4c8 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py @@ -9,38 +9,20 @@ from paramiko.ssh_exception import AuthenticationException from paramiko.ssh_exception import BadHostKeyException from paramiko.ssh_exception import SSHException +import requests +from retrying import retry from string import Template from django.conf import settings from git import Repo -from .base_vm_app import BaseVMAppPlugin +from .simple_web_app import SimpleWebAppPlugin from celery.utils.log import get_task_logger log = get_task_logger(__name__) -# Ansible playbook at this URL will be used to configure a bare-bones VM -ANSIBLE_PLAYBOOK_REPO = 'https://github.com/afgane/Rancher-Ansible' -INVENTORY_TEMPLATE = Template(""" -[Rancher] -rancher ansible_ssh_host=${master} - -[Agents] - -[cluster:children] -Rancher -Agents - -[cluster:vars] -ansible_ssh_port=22 -ansible_user='${user}' -ansible_ssh_private_key_file=pk -ansible_ssh_extra_args='-o StrictHostKeyChecking=no' -""") - - -class CloudMan2AppPlugin(BaseVMAppPlugin): +class CloudMan2AppPlugin(SimpleWebAppPlugin): """CloudLaunch appliance implementation for CloudMan 2.0.""" def __init__(self): @@ -79,6 +61,8 @@ def _remove_known_host(self, host): return True return False + @retry(retry_on_result=lambda result: result is False, wait_fixed=5000, + stop_max_delay=180000) def _check_ssh(self, host, pk=None, user='ubuntu'): """ Check for ssh availability on a host. @@ -86,12 +70,12 @@ def _check_ssh(self, host, pk=None, user='ubuntu'): :type host: ``str`` :param host: Hostname or IP address of the host to check. - :type user: ``str`` - :param user: Username to use when trying to login. - :type pk: ``str`` :param pk: Private portion of an ssh key. + :type user: ``str`` + :param user: Username to use when trying to login. + :rtype: ``bool`` :return: True if ssh connection was successful. """ @@ -99,6 +83,9 @@ def _check_ssh(self, host, pk=None, user='ubuntu'): ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) pkey = None if pk: + if 'RSA' not in pk: + # AWS at least does not specify key type yet paramiko requires + pk = pk.replace(' PRIVATE', ' RSA PRIVATE') key_file_object = StringIO(pk) pkey = paramiko.RSAKey.from_private_key(key_file_object) key_file_object.close() @@ -113,14 +100,23 @@ def _check_ssh(self, host, pk=None, user='ubuntu'): self._remove_known_host(host) return False - def _run_playbook(self, host, pk, user='ubuntu'): + def _run_playbook(self, playbook, inventory, host, pk, user='ubuntu'): """ Run an Ansible playbook to configure a host. - First clone an playbook repo if not already available, configure the - Ansible inventory and run the playbook. + First clone a playbook from the supplied repo if not already + available, configure the Ansible inventory, and run the playbook. + + The method assumes ``ansible-playbook`` system command is available. + + :type playbook: ``str`` + :param playbook: A URL of a git repository where the playbook resides. - The method assumes ``ansible-playbook`` command is available. + :type inventory: ``str`` + :param inventory: A URL pointing to a string ``Template``-like file + that will be used for running the playbook. The + file should have defined variables for ``host`` and + ``user``. :type host: ``str`` :param host: Hostname or IP of a machine as the playbook target. @@ -135,70 +131,51 @@ def _run_playbook(self, host, pk, user='ubuntu'): repo_path = './cloudlaunch/backend_plugins/rancher_ansible_%s' % host inventory_path = os.path.join(repo_path, 'inventory') # Ensure the playbook is available - log.info("Cloning Ansible playbook {0} to {1}".format( - ANSIBLE_PLAYBOOK_REPO, repo_path)) - Repo.clone_from(ANSIBLE_PLAYBOOK_REPO, to_path=repo_path) - # Create an inventory file + log.info("Cloning Ansible playbook %s to %s", playbook, repo_path) + Repo.clone_from(playbook, to_path=repo_path) + # Create a private ssh key file pkf = os.path.join(repo_path, 'pk') with os.fdopen(os.open(pkf, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as f: f.writelines(pk) + # Create an inventory file + r = requests.get(inventory) + inv = Template((r.content).decode('utf-8')) with open(inventory_path, 'w') as f: - log.info("Creating inventory file {0}".format(inventory_path)) - f.writelines(INVENTORY_TEMPLATE.substitute( - {'master': host, 'user': user})) + log.info("Creating inventory file %s", inventory_path) + f.writelines(inv.substitute({'host': host, 'user': user})) # Run the playbook cmd = "cd {0} && ansible-playbook -i inventory other.yml".format( repo_path) - log.info("Running Ansible with command {0}".format(cmd)) + log.info("Running Ansible with command %s", cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) - (out, err) = p.communicate() + (out, _) = p.communicate() p_status = p.wait() - log.info("Playbook stdout: %s\nstatus: %s" % (out, p_status)) - log.info("Deleting pk file {0} needed by Ansible".format(pkf)) + log.info("Playbook stdout: %s\nstatus: %s", out, p_status) if not settings.DEBUG: + log.info("Deleting ansible playbook %s", repo_path) shutil.rmtree(repo_path) return (p_status, out) - def launch_app(self, provider, task, name, cloud_config, - app_config, user_data): - """ - Handle the app launch process. - - This will: - - Perform necessary checks and env setup, most notably create a new - key pair. - - Launch an instance and wait until ssh access can be established - - Run an Ansible playbook to configure the instance - """ - # Implicitly create a new KP for this instance - # Note that this relies on the baseVMApp implementation! - kp_name = "CL-" + "".join([c for c in name if c.isalpha() or - c.isdigit()]).rstrip() - app_config['config_cloudlaunch']['keyPair'] = kp_name - # Launch an instance and check ssh connectivity - result = super(CloudMan2AppPlugin, self).launch_app( - provider, task, name, cloud_config, app_config, - user_data=None) - inst = provider.compute.instances.get( - result.get('cloudLaunch', {}).get('instance', {}).get('id')) - pk = result.get('cloudLaunch', {}).get('keyPair', {}).get('material') - timeout = 0 - while (not self._check_ssh(inst.public_ips[0], pk=pk) or - timeout > 200): - log.info("Waiting for ssh on {0}...".format(inst.name)) - time.sleep(5) - timeout += 5 - # Configure the instance + def _configure_host(self, name, task, app_config, provider_config): + log.debug("Running CloudMan2AppPlugin _configure_host for %s", name) + host = provider_config.get('host_address') + user = provider_config.get('ssh_user') + ssh_private_key = provider_config.get('ssh_private_key') + self._check_ssh(host, pk=ssh_private_key, user=user) task.update_state( state='PROGRESSING', meta={'action': 'Configuring container cluster manager.'}) - self._run_playbook(inst.public_ips[0], pk) - result['cloudLaunch']['applicationURL'] = \ - 'http://{0}:8080/'.format(result['cloudLaunch']['publicIP']) + playbook = app_config.get('config_appliance', {}).get('repository') + inventory = app_config.get( + 'config_appliance', {}).get('inventoryTemplate') + self._run_playbook(playbook, inventory, host, ssh_private_key, user) + result = {} + result['cloudLaunch'] = {'applicationURL': + 'http://{0}:8080/'.format(host)} task.update_state( state='PROGRESSING', meta={'action': "Waiting for CloudMan to become ready at %s" - % result['cloudLaunch']['applicationURL']}) + % result['cloudLaunch']['applicationURL']}) self.wait_for_http(result['cloudLaunch']['applicationURL']) return result diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman_app.py index 81c30259..9df2f737 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman_app.py @@ -1,10 +1,12 @@ import yaml + +from celery.utils.log import get_task_logger from urllib.parse import urlparse from rest_framework.serializers import ValidationError + from .simple_web_app import SimpleWebAppPlugin -import logging -log = logging.getLogger(__name__) +log = get_task_logger('cloudlaunch') def get_required_val(data, name, message): @@ -122,18 +124,24 @@ def sanitise_app_config(app_config): app_config['config_cloudman']['clusterPassword'] = '********' return app_config - def launch_app(self, provider, task, name, cloud_config, - app_config, user_data): - ud = yaml.dump(user_data, default_flow_style=False, allow_unicode=False) - # Make sure the placement and image ID (eg from a saved cluster) propagate + def deploy(self, name, task, app_config, provider_config): + """See the parent class in ``app_plugin.py`` for the docstring.""" + user_data = provider_config.get('cloud_user_data') + ud = yaml.dump(user_data, default_flow_style=False, + allow_unicode=False) + provider_config['cloud_user_data'] = ud + # Make sure the placement and image ID propagate + # (eg from a saved cluster) if user_data.get('placement'): - app_config.get('config_cloudlaunch')['placementZone'] = user_data['placement'] + app_config.get('config_cloudlaunch')[ + 'placementZone'] = user_data['placement'] if user_data.get('machine_image_id'): - app_config.get('config_cloudlaunch')['customImageID'] = user_data['machine_image_id'] - result = super(CloudManAppPlugin, self).launch_app( - provider, task, name, cloud_config, app_config, ud, check_http=False) - result['cloudLaunch']['applicationURL'] = \ - 'http://{0}/cloud'.format(result['cloudLaunch']['publicIP']) + app_config.get('config_cloudlaunch')[ + 'customImageID'] = user_data['machine_image_id'] + result = super(CloudManAppPlugin, self).deploy( + name, task, app_config, provider_config, check_http=False) + result['cloudLaunch']['applicationURL'] = 'http://{0}/cloud'.format( + result['cloudLaunch']['publicIP']) task.update_state( state='PROGRESSING', meta={'action': "Waiting for CloudMan to become ready at %s" diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/docker_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/docker_app.py index 9a56456d..f6e81f11 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/docker_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/docker_app.py @@ -52,9 +52,9 @@ def process_app_config(provider, name, cloud_config, app_config): user_data += " {0}".format(docker_config.get('repo_name')) return user_data - def launch_app(self, provider, task, name, cloud_config, - app_config, user_data): - result = super(DockerAppPlugin, self).launch_app( - provider, task, name, cloud_config, app_config, user_data) - result['cloudLaunch']['applicationURL'] = 'http://{0}'.format(result['cloudLaunch']['publicIP']) + def deploy(self, name, task, app_config, provider_config): + result = super(DockerAppPlugin, self).deploy( + name, task, app_config, provider_config) + result['cloudLaunch']['applicationURL'] = 'http://{0}'.format( + result['cloudLaunch']['publicIP']) return result diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/gvl_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/gvl_app.py index 9a56e5bd..cf099ab7 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/gvl_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/gvl_app.py @@ -31,9 +31,11 @@ def sanitise_app_config(app_config): sanitised_config['config_gvl'] = CloudManAppPlugin().sanitise_app_config(gvl_config) return sanitised_config - def launch_app(self, provider, task, name, cloud_config, - app_config, user_data): - ud = yaml.dump(user_data, default_flow_style=False, allow_unicode=False) - result = super(GVLAppPlugin, self).launch_app( - provider, task, name, cloud_config, app_config, ud) + def deploy(self, name, task, app_config, provider_config): + user_data = provider_config.get('cloud_user_data') + ud = yaml.dump(user_data, default_flow_style=False, + allow_unicode=False) + provider_config['cloud_user_data'] = ud + result = super(GVLAppPlugin, self).deploy( + name, task, app_config, provider_config) return result diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/simple_web_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/simple_web_app.py index 73008d35..ce7ee184 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/simple_web_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/simple_web_app.py @@ -1,11 +1,13 @@ """Plugin implementation for a simple web application.""" -import logging import time + +from celery.utils.log import get_task_logger import requests import requests.exceptions + from .base_vm_app import BaseVMAppPlugin -log = logging.getLogger(__name__) +log = get_task_logger('cloudlaunch') class SimpleWebAppPlugin(BaseVMAppPlugin): @@ -45,8 +47,7 @@ def wait_for_http(self, url, ok_status_codes=None, max_retries=200, pass count += 1 - def launch_app(self, provider, task, name, cloud_config, - app_config, user_data, **kwargs): + def deploy(self, name, task, app_config, provider_config, **kwargs): """ Handle the app launch process and wait for http. @@ -54,8 +55,8 @@ def launch_app(self, provider, task, name, cloud_config, want this method to perform the app http check and prefer to handle it in the child class. """ - result = super(SimpleWebAppPlugin, self).launch_app( - provider, task, name, cloud_config, app_config, user_data) + result = super(SimpleWebAppPlugin, self).deploy( + name, task, app_config, provider_config) check_http = kwargs.get('check_http', True) if check_http and result.get('cloudLaunch', {}).get('publicIP'): log.info("Simple web app going to wait for http") diff --git a/django-cloudlaunch/cloudlaunch/migrations/0004_add_user_profile.py b/django-cloudlaunch/cloudlaunch/migrations/0004_add_user_profile.py new file mode 100644 index 00000000..39a4ec47 --- /dev/null +++ b/django-cloudlaunch/cloudlaunch/migrations/0004_add_user_profile.py @@ -0,0 +1,43 @@ +# Generated by Django 2.0 on 2018-02-12 00:58 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('cloudlaunch', '0003_deployment_credentials_relationship'), + ] + + operations = [ + migrations.CreateModel( + name='PublicKey', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('added', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=60)), + ('public_key', models.TextField(max_length=16384)), + ('default', models.BooleanField(default=False, help_text='If set, use as the default public key')), + ('fingerprint', models.CharField(blank=True, max_length=100, null=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='UserProfile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='cloudlaunch_user_profile', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.AddField( + model_name='publickey', + name='user_profile', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='public_key', to='cloudlaunch.UserProfile'), + ), + ] diff --git a/django-cloudlaunch/cloudlaunch/models.py b/django-cloudlaunch/cloudlaunch/models.py index a36b7c76..b9fea2bc 100644 --- a/django-cloudlaunch/cloudlaunch/models.py +++ b/django-cloudlaunch/cloudlaunch/models.py @@ -325,3 +325,44 @@ class Usage(models.Model): class Meta: ordering = ['added'] verbose_name_plural = 'Usage' + + +class PublicKey(cb_models.DateNameAwareModel): + """Allow users to store their ssh public keys.""" + + public_key = models.TextField(max_length=16384) + default = models.BooleanField( + help_text="If set, use as the default public key", + blank=True, default=False) + # Ideally, we would auto-generate the fingerprint from the public key + # instead of prompting the user for it but AWS at least uses two different + # methods of generating it, making the autogeneration impractical: + # http://bit.ly/2EIs0kR + fingerprint = models.CharField(max_length=100, blank=True, null=True) + user_profile = models.ForeignKey('UserProfile', models.CASCADE, + related_name='public_key') + + def save(self, *args, **kwargs): + # Ensure only 1 public key is selected as the 'default' + # This is not atomic but don't know how to enforce it at the + # DB level directly. + if self.default is True: + previous_default = PublicKey.objects.filter( + default=True, user_profile=self.user_profile).first() + if previous_default: + previous_default.default = False + previous_default.save() + return super(PublicKey, self).save() + + +class UserProfile(models.Model): + """User profile specific to CloudLaunch.""" + + # Link UserProfile to a User model instance + user = models.OneToOneField( + User, models.CASCADE, related_name="cloudlaunch_user_profile") + + def __str__(self): + """Set default display for objects.""" + return "{0} ({1} {2})".format(self.user.username, self.user.first_name, + self.user.last_name) diff --git a/django-cloudlaunch/cloudlaunch/serializers.py b/django-cloudlaunch/cloudlaunch/serializers.py index d4e86cbf..c7c1a2d2 100644 --- a/django-cloudlaunch/cloudlaunch/serializers.py +++ b/django-cloudlaunch/cloudlaunch/serializers.py @@ -1,5 +1,6 @@ import json import jsonmerge +import logging from bioblend.cloudman.launch import CloudManLauncher from cloudbridge.cloud.factory import ProviderList @@ -14,6 +15,8 @@ from djcloudbridge import view_helpers from djcloudbridge.drf_helpers import CustomHyperlinkedIdentityField +log = logging.getLogger(__name__) + class CloudManSerializer(serializers.Serializer): """ @@ -140,27 +143,25 @@ def create(self, validated_data): :param validated_data: Dict containing action the task should perform. Valid actions are `HEALTH_CHECK`, `DELETE`. """ - print("deployment task data: %s" % validated_data) + log.debug("Deployment task data: %s", validated_data) action = getattr(models.ApplicationDeploymentTask, validated_data.get( - 'action', - models.ApplicationDeploymentTask.HEALTH_CHECK)) + 'action', + models.ApplicationDeploymentTask.HEALTH_CHECK)) request = self.context.get('view').request dpk = self.context['view'].kwargs.get('deployment_pk') dpl = models.ApplicationDeployment.objects.get(id=dpk) creds = (cb_models.Credentials.objects.get_subclass( - id=dpl.credentials.id).as_dict() + id=dpl.credentials.id).as_dict() if dpl.credentials else view_helpers.get_credentials(dpl.target_cloud, request)) try: if action == models.ApplicationDeploymentTask.HEALTH_CHECK: async_result = tasks.health_check.delay(dpl.pk, creds) elif action == models.ApplicationDeploymentTask.RESTART: - async_result = tasks.restart_appliance.delay(dpl.pk, - creds) + async_result = tasks.restart_appliance.delay(dpl.pk, creds) elif action == models.ApplicationDeploymentTask.DELETE: - async_result = tasks.delete_appliance.delay(dpl.pk, - creds) + async_result = tasks.delete_appliance.delay(dpl.pk, creds) return models.ApplicationDeploymentTask.objects.create( action=action, deployment=dpl, celery_id=async_result.task_id) except serializers.ValidationError as ve: @@ -250,20 +251,21 @@ def create(self, validated_data): handler = util.import_class(version.backend_component_name)() app_config = validated_data.get("config_app", {}) - merged_config = jsonmerge.merge(default_combined_config, app_config) + merged_app_config = jsonmerge.merge( + default_combined_config, app_config) cloud_config = util.serialize_cloud_config(cloud_version_config) final_ud_config = handler.process_app_config( - provider, name, cloud_config, merged_config) - sanitised_app_config = handler.sanitise_app_config(merged_config) - async_result = tasks.launch_appliance.delay( - name, cloud_version_config.pk, credentials, merged_config, + provider, name, cloud_config, merged_app_config) + sanitised_app_config = handler.sanitise_app_config(merged_app_config) + async_result = tasks.create_appliance.delay( + name, cloud_version_config.pk, credentials, merged_app_config, final_ud_config) del validated_data['application'] if 'config_app' in validated_data: del validated_data['config_app'] validated_data['owner_id'] = request.user.id - validated_data['application_config'] = json.dumps(merged_config) + validated_data['application_config'] = json.dumps(merged_app_config) validated_data['credentials_id'] = credentials.get('id') or None app_deployment = super(DeploymentSerializer, self).create(validated_data) self.log_usage(cloud_version_config, app_deployment, sanitised_app_config, request.user) @@ -287,3 +289,18 @@ def log_usage(self, app_version_cloud_config, app_deployment, sanitised_app_conf u = models.Usage(app_version_cloud_config=app_version_cloud_config, app_deployment=app_deployment, app_config=sanitised_app_config, user=user) u.save() + + +class PublicKeySerializer(serializers.HyperlinkedModelSerializer): + url = serializers.HyperlinkedIdentityField( + view_name='public-key-detail', read_only=True) + + class Meta: + model = models.PublicKey + exclude = ['user_profile'] + + def create(self, validated_data): + user_profile, _ = models.UserProfile.objects.get_or_create( + user=self.context.get('view').request.user) + return models.PublicKey.objects.create( + user_profile=user_profile, **validated_data) diff --git a/django-cloudlaunch/cloudlaunch/tasks.py b/django-cloudlaunch/cloudlaunch/tasks.py index ccdd75da..f26ada1d 100644 --- a/django-cloudlaunch/cloudlaunch/tasks.py +++ b/django-cloudlaunch/cloudlaunch/tasks.py @@ -1,10 +1,9 @@ """Tasks to be executed asynchronously (via Celery).""" import copy import json -import traceback +import logging from celery.app import shared_task -from celery.exceptions import Ignore from celery.exceptions import SoftTimeLimitExceeded from celery.result import AsyncResult from celery.utils.log import get_task_logger @@ -14,7 +13,11 @@ from . import signals from . import util -LOG = get_task_logger(__name__) +log = get_task_logger('cloudlaunch') +# Limit how much these libraries log +logging.getLogger('boto3').setLevel(logging.WARNING) +logging.getLogger('botocore').setLevel(logging.WARNING) +logging.getLogger('cloudbridge').setLevel(logging.INFO) @shared_task(time_limit=120) @@ -42,32 +45,40 @@ def migrate_launch_task(task_id): @shared_task(expires=120) -def launch_appliance(name, cloud_version_config_id, credentials, app_config, - user_data, task_id=None): +def create_appliance(name, cloud_version_config_id, credentials, app_config, + user_data): """Call the appropriate app plugin and initiate the app launch process.""" - launch_result = {} try: - LOG.debug("Launching appliance %s", name) - cloud_version_config = models.ApplicationVersionCloudConfig.objects.get( + log.debug("Creating appliance %s", name) + cloud_version_conf = models.ApplicationVersionCloudConfig.objects.get( pk=cloud_version_config_id) plugin = util.import_class( - cloud_version_config.application_version.backend_component_name)() - provider = domain_model.get_cloud_provider(cloud_version_config.cloud, - credentials) - cloud_config = util.serialize_cloud_config(cloud_version_config) - LOG.info("Launching app %s with the follwing app config: %s \n and " - "cloud config: %s", name, app_config, cloud_config) - launch_result = plugin.launch_app(provider, Task(launch_appliance), - name, cloud_config, app_config, - user_data) + cloud_version_conf.application_version.backend_component_name)() + provider = domain_model.get_cloud_provider( + cloud_version_conf.cloud, credentials) + cloud_config = util.serialize_cloud_config(cloud_version_conf) + # TODO: Add keys (& support) for using existing, user-supplied hosts + provider_config = {'cloud_provider': provider, + 'cloud_config': cloud_config, + 'cloud_user_data': user_data} + log.info("Provider_config: %s", provider_config) + log.info("Creating app %s with the follwing app config: %s \n and " + "cloud config: %s", name, app_config, provider_config) + deploy_result = plugin.deploy(name, Task(create_appliance), app_config, + provider_config) # Schedule a task to migrate result one hour from now - migrate_launch_task.apply_async([launch_appliance.request.id], + migrate_launch_task.apply_async([create_appliance.request.id], countdown=3600) - return launch_result + return deploy_result except SoftTimeLimitExceeded: - raise Exception("Launch task time limit exceeded; stopping the task.") - except Exception as e: - raise Exception("Launch task failed: %s" % str(e)) from e + msg = "Create appliance task time limit exceeded; stopping the task." + log.warning(msg) + raise Exception(msg) + except Exception as exc: + msg = "Create appliance task failed: %s" % str(exc) + log.error(msg) + raise Exception(msg) from exc + def _get_app_plugin(deployment): """ @@ -87,7 +98,7 @@ def _get_app_plugin(deployment): @shared_task(time_limit=120) def migrate_task_result(task_id): """Migrate task results to the database from the broker table.""" - LOG.debug("Migrating task %s result to the DB" % task_id) + log.debug("Migrating task %s result to the DB" % task_id) adt = models.ApplicationDeploymentTask.objects.get(celery_id=task_id) task = AsyncResult(task_id) task_meta = task.backend.get_task_meta(task.id) @@ -131,14 +142,16 @@ def health_check(self, deployment_id, credentials): """ try: deployment = models.ApplicationDeployment.objects.get(pk=deployment_id) - LOG.debug("Checking health of deployment %s", deployment.name) + log.debug("Checking health of deployment %s", deployment.name) plugin = _get_app_plugin(deployment) dpl = _serialize_deployment(deployment) provider = domain_model.get_cloud_provider(deployment.target_cloud, credentials) result = plugin.health_check(provider, dpl) except Exception as e: - raise Exception("Health check failed: %s" % str(e)) from e + msg = "Health check failed: %s" % str(e) + log.error(msg) + raise Exception(msg) from e finally: # We only keep the two most recent health check task results so delete # any older ones @@ -157,14 +170,16 @@ def restart_appliance(self, deployment_id, credentials): """ try: deployment = models.ApplicationDeployment.objects.get(pk=deployment_id) - LOG.debug("Performing restart on deployment %s", deployment.name) + log.debug("Performing restart on deployment %s", deployment.name) plugin = _get_app_plugin(deployment) dpl = _serialize_deployment(deployment) provider = domain_model.get_cloud_provider(deployment.target_cloud, credentials) result = plugin.restart(provider, dpl) except Exception as e: - raise Exception("Restart task failed: %s" % str(e)) from e + msg = "Restart task failed: %s" % str(e) + log.error(msg) + raise Exception(msg) from e # Schedule a task to migrate results right after task completion # Do this as a separate task because until this task completes, we # cannot obtain final status or traceback. @@ -181,7 +196,7 @@ def delete_appliance(self, deployment_id, credentials): """ try: deployment = models.ApplicationDeployment.objects.get(pk=deployment_id) - LOG.debug("Performing delete on deployment %s", deployment.name) + log.debug("Performing delete on deployment %s", deployment.name) plugin = _get_app_plugin(deployment) dpl = _serialize_deployment(deployment) provider = domain_model.get_cloud_provider(deployment.target_cloud, @@ -191,7 +206,9 @@ def delete_appliance(self, deployment_id, credentials): deployment.archived = True deployment.save() except Exception as e: - raise Exception("Delete task failed: %s" % str(e)) from e + msg = "Delete task failed: %s" % str(e) + log.error(msg) + raise Exception(msg) from e # Schedule a task to migrate results right after task completion # Do this as a separate task because until this task completes, we # cannot obtain final status or traceback. diff --git a/django-cloudlaunch/cloudlaunch/urls.py b/django-cloudlaunch/cloudlaunch/urls.py index e65b438c..817cc044 100644 --- a/django-cloudlaunch/cloudlaunch/urls.py +++ b/django-cloudlaunch/cloudlaunch/urls.py @@ -50,7 +50,7 @@ schema_view = get_schema_view(title='CloudLaunch API') urlpatterns = [ - url(r'api/v1/auth/api-token-auth/', views.AuthTokenView.as_view()), + url(r'%sapi-token-auth/' % auth_regex_pattern, views.AuthTokenView.as_view()), url(r'api/v1/', include(router.urls)), url(r'api/v1/', include(deployments_router.urls)), # This generates a duplicate url set with the cloudman url included @@ -58,11 +58,16 @@ url(infrastructure_regex_pattern, include(cloud_router.get_urls())), url(infrastructure_regex_pattern, include('djcloudbridge.urls')), url(auth_regex_pattern, include(('rest_auth.urls', 'rest_auth'), namespace='rest_auth')), - url(r'api/v1/auth/registration', include(('rest_auth.registration.urls', 'rest_auth_reg'), + url(r'%sregistration' % auth_regex_pattern, include(('rest_auth.registration.urls', 'rest_auth_reg'), namespace='rest_auth_reg')), + url(r'%suser/public-keys/$' % + auth_regex_pattern, views.PublicKeyList.as_view()), + url(r'%suser/public-keys/(?P[0-9]+)/$' % + auth_regex_pattern, views.PublicKeyDetail.as_view(), + name='public-key-detail'), url(auth_regex_pattern, include(('rest_framework.urls', 'rest_framework'), namespace='rest_framework')), - url(r'api/v1/auth/', include('djcloudbridge.profile.urls')), + url(auth_regex_pattern, include('djcloudbridge.profile.urls')), # The following is required because rest_auth calls allauth internally and # reverse urls need to be resolved. url(r'accounts/', include('allauth.urls')), diff --git a/django-cloudlaunch/cloudlaunch/views.py b/django-cloudlaunch/cloudlaunch/views.py index 8efbbaa5..caedd52f 100644 --- a/django-cloudlaunch/cloudlaunch/views.py +++ b/django-cloudlaunch/cloudlaunch/views.py @@ -4,6 +4,7 @@ from django_filters import rest_framework as dj_filters from rest_framework import authentication from rest_framework import filters +from rest_framework import generics from rest_framework import mixins from rest_framework import permissions from rest_framework import renderers @@ -136,3 +137,26 @@ def get_queryset(self): user = self.request.user return models.ApplicationDeploymentTask.objects.filter( deployment=deployment, deployment__owner=user) + + +class PublicKeyList(generics.ListCreateAPIView): + """List public ssh keys associated with the user profile.""" + + permission_classes = (IsAuthenticated,) + serializer_class = serializers.PublicKeySerializer + + def get_queryset(self): + c.incr('user.list.public.keys') + return models.PublicKey.objects.filter( + user_profile__user=self.request.user) + + +class PublicKeyDetail(generics.RetrieveUpdateDestroyAPIView): + """Get a single public ssh keys associated with the user profile.""" + + permission_classes = (IsAuthenticated,) + serializer_class = serializers.PublicKeySerializer + + def get_queryset(self): + return models.PublicKey.objects.filter( + user_profile__user=self.request.user) diff --git a/django-cloudlaunch/cloudlaunchserver/celery.py b/django-cloudlaunch/cloudlaunchserver/celery.py index d0c2285c..c5f23d67 100644 --- a/django-cloudlaunch/cloudlaunchserver/celery.py +++ b/django-cloudlaunch/cloudlaunchserver/celery.py @@ -3,9 +3,11 @@ from __future__ import absolute_import import os +import raven -from celery import Celery +import celery from django.conf import settings # noqa +from raven.contrib.celery import register_signal, register_logger_signal import logging log = logging.getLogger(__name__) @@ -13,11 +15,20 @@ # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cloudlaunchserver.settings') -app = Celery('proj') -# Using a string here means the worker will not have to -# pickle the object when using Windows. +class Celery(celery.Celery): + + def on_configure(self): + client = raven.Client(settings.RAVEN_CONFIG.get('dsn')) + + # register a custom filter to filter out duplicate logs + register_logger_signal(client) + # hook into the Celery error handler + register_signal(client) + + +app = Celery('proj') # Changed to use dedicated celery config as detailed in: # http://docs.celeryproject.org/en/latest/getting-started/first-steps-with-celery.html # app.config_from_object('django.conf:settings') diff --git a/django-cloudlaunch/cloudlaunchserver/settings.py b/django-cloudlaunch/cloudlaunchserver/settings.py index 0db23c42..0fb7393f 100644 --- a/django-cloudlaunch/cloudlaunchserver/settings.py +++ b/django-cloudlaunch/cloudlaunchserver/settings.py @@ -11,6 +11,7 @@ """ from django.conf import settings as django_settings import os +import raven import sys # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -100,7 +101,8 @@ 'django_celery_results', 'django_celery_beat', 'django_countries', - 'django_filters' + 'django_filters', + 'raven.contrib.django.raven_compat' ] MIDDLEWARE = [ @@ -222,12 +224,16 @@ } REST_SESSION_LOGIN = True +RAVEN_CONFIG = { + 'dsn': os.environ.get('SENTRY_DSN', 'your_sentry_dsn') +} + LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'verbose': { - 'format': '[%(asctime)s %(name)s:%(lineno)d %(levelname)s] %(message)s' + 'format': '%(asctime)s %(levelname)s %(pathname)s:%(lineno)d - %(message)s' }, }, 'filters': { @@ -242,24 +248,47 @@ 'formatter': 'verbose', 'level': 'DEBUG', }, - 'file': { + 'file-cloudlaunch': { 'class': 'logging.FileHandler', + 'formatter': 'verbose', + 'level': 'INFO', 'filename': 'cloudlaunch.log', + }, + 'file-django': { + 'class': 'logging.FileHandler', 'formatter': 'verbose', + 'level': 'WARNING', + 'filename': 'cloudlaunch-django.log', }, + 'sentry': { + 'formatter': 'verbose', + 'level': 'WARNING', + 'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler' + } }, 'loggers': { 'django': { - 'handlers': ['console'], - 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + 'handlers': ['console', 'file-django', 'sentry'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), }, - 'cloudlaunch': { - 'handlers': ['console', 'file'], + 'django.db.backends': { + 'handlers': ['file-django', 'sentry'], 'level': 'INFO', }, - 'django.db.backends': { + 'django.template': { + 'handlers': ['console', 'file-django', 'sentry'], 'level': 'INFO', - 'handlers': ['console'], + 'propagate': True, + }, + 'django.server': { + 'handlers': ['console', 'file-django', 'sentry'], + 'level': 'ERROR', + 'propagate': True, + }, + 'cloudlaunch': { + 'handlers': ['console', 'file-cloudlaunch', 'sentry'], + 'level': 'DEBUG', + 'propagate': False }, }, } diff --git a/requirements.txt b/requirements.txt index db0afe56..3fb789c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ git+git://github.com/gvlproject/cloudbridge#egg=cloudbridge git+git://github.com/CloudVE/djcloudbridge#egg=djcloudbridge --e ".[prod]" \ No newline at end of file +-e ".[prod]" diff --git a/setup.py b/setup.py index 1e5fa7be..92c15f1a 100755 --- a/setup.py +++ b/setup.py @@ -85,7 +85,9 @@ def get_version(*file_paths): # For merging userdata/config dictionaries 'jsonmerge>=1.4.0', # For commandline option handling - 'click' + 'click', + # Integration with Sentry + 'raven' ] REQS_PROD = ([ @@ -105,7 +107,8 @@ def get_version(*file_paths): # As celery message broker during development 'redis', 'sphinx>=1.3.1', - 'bumpversion>=0.5.3'] + REQS_TEST + 'bumpversion>=0.5.3', + 'pylint-django'] + REQS_TEST ) setup( diff --git a/tox.ini b/tox.ini index f7c45bbe..5d3d17cb 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ skipsdist = True [testenv] commands = {envpython} -m coverage run --source . --branch manage.py test changedir=django-cloudlaunch +passenv = + SENTRY_DSN deps = -rrequirements.txt coverage