Merge pull request #1497 from mfalesni/trackerbot-check-upd

Stable Sprout

sprout/appliances/admin.py

@@ -175,4 +175,5 @@ def show_ip_address(self, instance):
 
 @register_for(Template)
 class TemplateAdmin(Admin):
-    list_display = ["name", "version", "original_name", "ready", "exists", "date", "template_group"]
+    list_display = [
+        "name", "version", "original_name", "ready", "exists", "date", "template_group", "usable"]

sprout/appliances/migrations/0005_template_usable.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('appliances', '0004_provider_appliance_limit'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='template',
+            name='usable',
+            field=models.BooleanField(default=False, help_text=b'Template is marked as usable'),
+            preserve_default=True,
+        ),
+    ]

sprout/appliances/models.py

@@ -11,6 +11,8 @@
 from utils.providers import provider_factory
 from utils.version import LooseVersion
 
+from sprout import redis
+
 
 def logger():
     return create_logger("sprout")
@@ -130,6 +132,7 @@ class Template(models.Model):
     status_changed = models.DateTimeField(auto_now_add=True)
     ready = models.BooleanField(default=False, help_text="Template is ready-to-be-used")
     exists = models.BooleanField(default=True, help_text="Template exists in the provider.")
+    usable = models.BooleanField(default=False, help_text="Template is marked as usable")
 
     @property
     def provider_api(self):
@@ -286,21 +289,22 @@ def give_to_pool(cls, pool, time_minutes):
             for template in pool.possible_templates:
                 for appliance in cls.unassigned().filter(
                         template=template).all()[:pool.total_count - n_appliances]:
-                    appliance.appliance_pool = pool
-                    appliance.datetime_leased = timezone.now()
-                    appliance.leased_until = appliance.datetime_leased + timedelta(
-                        minutes=time_minutes)
-                    if appliance.provider_api.can_rename:
-                        new_name = "{}_{}".format(pool.owner.username, appliance.name)
-                        try:
-                            appliance.provider_api.rename_vm(appliance.name, new_name)
-                        except Exception as e:
-                            logger().exception(str(e))
-                        else:
-                            appliance.name = new_name
-                    appliance.save()
-                    appliance_power_on.delay(appliance.id)
-                    n_appliances += 1
+                    new_name = "{}_{}".format(pool.owner.username, appliance.name)
+                    with redis.appliances_ignored_when_renaming(appliance.name, new_name):
+                        appliance.appliance_pool = pool
+                        appliance.datetime_leased = timezone.now()
+                        appliance.leased_until = appliance.datetime_leased + timedelta(
+                            minutes=time_minutes)
+                        if appliance.provider_api.can_rename:
+                            try:
+                                appliance.name = appliance.provider_api.rename_vm(
+                                    appliance.name, new_name)
+                            except Exception as e:
+                                logger().exception(
+                                    "Exception {}: {}".format(type(e).__name__, str(e)))
+                        appliance.save()
+                        appliance_power_on.delay(appliance.id)
+                        n_appliances += 1
                 if n_appliances == pool.total_count:
                     break
         return n_appliances
@@ -392,14 +396,15 @@ def create(cls, owner, group, version=None, date=None, num_appliances=1, time_le
         from appliances.tasks import request_appliance_pool
         # Retrieve latest possible
         if not version:
-            versions = Template.get_versions(template_group=group, ready=True)
+            versions = Template.get_versions(template_group=group, ready=True, usable=True)
             if versions:
                 version = versions[0]
         if not date:
             if version is not None:
-                dates = Template.get_dates(template_group=group, version=version, ready=True)
+                dates = Template.get_dates(template_group=group, version=version, ready=True,
+                    usable=True)
             else:
-                dates = Template.get_dates(template_group=group, ready=True)
+                dates = Template.get_dates(template_group=group, ready=True, usable=True)
             if dates:
                 date = dates[0]
         if isinstance(group, basestring):
@@ -430,7 +435,7 @@ def possible_templates(self):
         if self.date is not None:
             filter_params["date"] = self.date
         return Template.objects.filter(
-            template_group=self.group, ready=True, exists=True, **filter_params).all()
+            template_group=self.group, ready=True, exists=True, usable=True, **filter_params).all()
 
     @property
     def possible_provisioning_templates(self):

sprout/appliances/tasks.py

@@ -14,7 +14,7 @@
 
 from appliances.models import (
     Provider, Group, Template, Appliance, AppliancePool, DelayedProvisionTask)
-from sprout import settings
+from sprout import settings, redis
 from sprout.celery import app as celery_app
 
 from utils.appliance import Appliance as CFMEAppliance
@@ -194,27 +194,49 @@ def poke_trackerbot(self):
     """This beat-scheduled task periodically polls the trackerbot if there are any new templates.
     """
     try:
-        for template in trackerbot().template().get()["objects"]:
-            try:
-                group = template["group"]["name"]
-            except KeyError:
+        template_usability = []
+        for template in trackerbot().providertemplate().get(limit=10000)["objects"]:
+            template_usability.append(
+                (
+                    template["provider"]["key"],
+                    template["template"]["name"],
+                    template["tested"] and template["usable"]
+                )
+            )
+            if not template["tested"] or not template["usable"]:
                 continue
-            template_name = template["name"]
-            group, created = Group.objects.get_or_create(id=group)
-            for provider in template["usable_providers"]:
-                provider, created = Provider.objects.get_or_create(id=provider)
-                if not provider.working:
-                    continue
-                if "sprout" not in provider.provider_data:
-                    continue
-                if not provider.provider_data.get("use_for_sprout", False):
-                    continue
-                try:
-                    Template.objects.get(
-                        provider=provider, template_group=group, original_name=template_name)
-                except ObjectDoesNotExist:
-                    if provider.api.does_vm_exist(template_name):
-                        create_appliance_template.delay(provider.id, group.id, template_name)
+            group, create = Group.objects.get_or_create(id=template["template"]["group"]["name"])
+            provider, create = Provider.objects.get_or_create(id=template["provider"]["key"])
+            if not provider.working:
+                continue
+            if "sprout" not in provider.provider_data:
+                continue
+            if not provider.provider_data.get("use_for_sprout", False):
+                continue
+            template_name = template["template"]["name"]
+            try:
+                Template.objects.get(
+                    provider=provider, template_group=group, original_name=template_name)
+            except ObjectDoesNotExist:
+                if provider.api.does_vm_exist(template_name):
+                    create_appliance_template.delay(provider.id, group.id, template_name)
+        # If any of the templates becomes unusable, let sprout know about it
+        # Similarly if some of them becomes usable ...
+        for provider_id, template_name, usability in template_usability:
+            provider, create = Provider.objects.get_or_create(id=provider_id)
+            try:
+                with transaction.atomic():
+                    template = Template.objects.get(provider=provider, original_name=template_name)
+                    template.usable = usability
+                    template.save()
+                    # Kill all shepherd appliances if they were acidentally spun up
+                    if not usability:
+                        for appliance in Appliance.objects.filter(
+                                template=template, ready=True, marked_for_deletion=False,
+                                appliance_pool=None):
+                            Appliance.kill(appliance)
+            except ObjectDoesNotExist:
+                pass
     finally:
         self.apply_async(countdown=600)
 
@@ -262,6 +284,7 @@ def prepare_template_deploy(self, template_id):
         if not template.is_created:
             template.set_status("Deploying the template.")
             kwargs = template.provider.provider_data["sprout"]
+            kwargs["power_on"] = True
             template.provider_api.deploy_template(
                 template.original_name, vm_name=template.name, **kwargs)
         else:
@@ -413,7 +436,7 @@ def replace_clone_to_pool(
     appliance_pool = AppliancePool.objects.get(id=appliance_pool_id)
     exclude_template = Template.objects.get(id=exclude_template_id)
     templates = Template.objects.filter(
-        ready=True, exists=True, template_group=appliance_pool.group, version=version,
+        ready=True, exists=True, usable=True, template_group=appliance_pool.group, version=version,
         date=date).all()
     templates_excluded = filter(lambda tpl: tpl != exclude_template, templates)
     if templates_excluded:
@@ -684,7 +707,8 @@ def retrieve_appliances_power_states(self):
     it schedules itself for execution again after some time"""
     try:
         for a in Appliance.objects.all():
-            a.retrieve_power_state()
+            if a.name not in redis.renaming_appliances:
+                a.retrieve_power_state()
     finally:
         self.apply_async(countdown=25)
 
@@ -716,6 +740,8 @@ def delete_nonexistent_appliances(self):
     """Goes through orphaned appliances' objects and deletes them from the database."""
     try:
         for appliance in Appliance.objects.filter(ready=True).all():
+            if appliance.name in redis.renaming_appliances:
+                continue
             if appliance.power_state == Appliance.Power.ORPHANED:
                 try:
                     appliance.delete()
@@ -754,8 +780,8 @@ def free_appliance_shepherd(self):
     running template's appliances and spins up new ones."""
     try:
         for grp in Group.objects.all():
-            group_versions = Template.get_versions(template_group=grp, ready=True)
-            group_dates = Template.get_dates(template_group=grp, ready=True)
+            group_versions = Template.get_versions(template_group=grp, ready=True, usable=True)
+            group_dates = Template.get_dates(template_group=grp, ready=True, usable=True)
             if group_versions:
                 # Downstream - by version (downstream releases)
                 filter_keep = {"version": group_versions[0]}
@@ -771,7 +797,8 @@ def free_appliance_shepherd(self):
             # Retrieve list of all templates for given group
             # I know joins might be a bit better solution but I'll leave that for later.
             possible_templates = list(
-                Template.objects.filter(ready=True, template_group=grp, **filter_keep).all())
+                Template.objects.filter(
+                    usable=True, ready=True, template_group=grp, **filter_keep).all())
             # If it can be deployed, it must exist
             possible_templates_for_provision = filter(lambda tpl: tpl.exists, possible_templates)
             appliances = []
@@ -809,7 +836,7 @@ def free_appliance_shepherd(self):
             # Killing old appliances
             for filter_kill in filters_kill:
                 for template in Template.objects.filter(
-                        ready=True, template_group=grp, **filter_kill):
+                        ready=True, usable=True, template_group=grp, **filter_kill):
                     for a in Appliance.objects.filter(
                             template=template, appliance_pool=None, marked_for_deletion=False):
                         Appliance.kill(a)

sprout/appliances/templates/appliances/_versions.html

@@ -1,4 +1,4 @@
 <option value="latest">{% if group %}Latest for {{ group.id }}{% else %}Latest{%endif %}</option>
-{% for version in versions %}
-<option value="{{version}}">{{ version }}</option>
+{% for version, providers in versions %}
+<option value="{{version}}">{{ version }}{% if providers %} ({{ providers }}){% endif %}</option>
 {% endfor %}

sprout/appliances/templates/index.html

@@ -22,6 +22,14 @@ <h2>Welcome to Sprout</h2>
         </p>
     {% else %}
         <p><strong>If you want to continue, please log in.</strong></p>
+        <p>
+            If you need to reset your password or you believe you shall have access into Sprout, you can contact any of the superusers:
+            <ul>
+                {% for superuser in superusers %}
+                    <li>{{ superuser.first_name }} {{ superuser.last_name }} {% if superuser.email %}&lt;{{ superuser.email }}&gt;{% endif %}</li>
+                {% endfor %}
+            </ul>
+        </p>
     {% endif %}
 </div>
 {% endblock %}

sprout/appliances/views.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from django.contrib import messages
+from django.contrib.auth.models import User
 from django.core.exceptions import ObjectDoesNotExist
 from django.http import HttpResponse
 from django.shortcuts import render, redirect
@@ -28,6 +29,7 @@ def go_back_or_home(request):
 
 
 def index(request):
+    superusers = User.objects.filter(is_superuser=True)
     return render(request, 'index.html', locals())
 
 
@@ -53,7 +55,15 @@ def versions_for_group(request):
         except ObjectDoesNotExist:
             versions = []
         else:
-            versions = Template.get_versions(template_group=group, ready=True)
+            versions_only = Template.get_versions(template_group=group, ready=True, usable=True)
+            versions = []
+            for version in versions_only:
+                providers = []
+                for provider in Template.objects.filter(
+                        version=version, usable=True, ready=True).values("provider"):
+                    providers.append(provider.values()[0])
+                versions.append((version, ", ".join(providers)))
+
     return render(request, 'appliances/_versions.html', locals())
 
 

sprout/requirements.txt

@@ -1,7 +1,7 @@
 -r ../requirements.txt
 Django
 django-object-actions
-celery[redis]
+celery[redis]<3.2
 honcho
 python-memcached
 flower