Merge pull request #250 from clarkperkins/feature/formula-cloning

Fix formula cloning

stackdio/api/formulas/api.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+from __future__ import unicode_literals
 
 import logging
 from collections import OrderedDict
@@ -26,6 +27,7 @@
 from rest_framework.serializers import ValidationError
 
 from stackdio.core.permissions import StackdioModelPermissions, StackdioObjectPermissions
+from stackdio.core.utils import recursively_sort_dict
 from stackdio.core.viewsets import (
     StackdioModelUserPermissionsViewSet,
     StackdioModelGroupPermissionsViewSet,
@@ -42,8 +44,7 @@ class FormulaListAPIView(generics.ListCreateAPIView):
     """
     Displays a list of all formulas visible to you.
     You may import a formula here also by providing a URI to a git repository containing a valid
-    SPECFILE at the root of the repo.  You may optionally provide a git_username and git_password
-    if your repository requires authentication.
+    SPECFILE at the root of the repo.
     """
     queryset = models.Formula.objects.all()
     serializer_class = serializers.FormulaSerializer
@@ -82,9 +83,21 @@ def perform_destroy(self, instance):
 
 class FormulaPropertiesAPIView(generics.RetrieveAPIView):
     queryset = models.Formula.objects.all()
-    serializer_class = serializers.FormulaPropertiesSerializer
     permission_classes = (StackdioObjectPermissions,)
 
+    def retrieve(self, request, *args, **kwargs):
+        formula = self.get_object()
+
+        # determine if a version was specified
+        version = request.query_params.get('version')
+
+        if version not in formula.get_valid_versions():
+            version = formula.default_version
+
+        properties = formula.properties(version)
+
+        return Response(recursively_sort_dict(properties))
+
 
 class FormulaComponentListAPIView(mixins.FormulaRelatedMixin, generics.ListAPIView):
     """
@@ -97,17 +110,10 @@ def list(self, request, *args, **kwargs):
         # determine if a version was specified
         version = request.query_params.get('version')
 
-        if version in formula.get_valid_versions():
-            components = formula.components_for_version(version)
-        else:
+        if version not in formula.get_valid_versions():
             version = formula.default_version
-            if formula.repo is None:
-                components = {}
-            else:
-                formula.repo.git.checkout(version)
-                components = formula.components
 
-        components = components.values()
+        components = formula.components(version).values()
 
         data = OrderedDict((
             ('count', len(components)),
@@ -125,7 +131,7 @@ class FormulaValidVersionListAPIView(mixins.FormulaRelatedMixin, generics.ListAP
     def list(self, request, *args, **kwargs):
         formula = self.get_formula()
 
-        versions = formula.get_valid_versions()
+        versions = sorted(formula.get_valid_versions())
 
         data = OrderedDict((
             ('count', len(versions)),

stackdio/api/formulas/exceptions.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2016,  Digital Reasoning
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+class FormulaTaskException(Exception):
+    pass
+
+
+class InvalidFormula(Exception):
+    pass
+
+
+class InvalidFormulaComponent(InvalidFormula):
+    pass

stackdio/api/formulas/filters.py

@@ -24,7 +24,6 @@
 
 class FormulaFilter(django_filters.FilterSet):
     title = django_filters.CharFilter(lookup_type='icontains')
-    component = django_filters.CharFilter(name='components__sls_path')
     q = OrFieldsFilter(field_names=('title', 'description', 'uri', 'root_path'),
                        lookup_type='icontains')
 
@@ -34,6 +33,5 @@ class Meta:
             'title',
             'uri',
             'root_path',
-            'component',
             'q',
         )

stackdio/api/formulas/migrations/0002_0_8_0_migrations.py

@@ -2,11 +2,62 @@
 # Generated by Django 1.9.5 on 2016-04-08 18:19
 from __future__ import unicode_literals
 
-from django.db import migrations
+from django.db import migrations, models
+from six.moves.urllib_parse import urljoin, urlsplit  # pylint: disable=import-error
 import django_extensions.db.fields
 import model_utils.fields
 
 
+def add_user_to_uri(apps, schema_editor):
+    Formula = apps.get_model('formulas', 'Formula')
+
+    for formula in Formula.objects.all():
+        if not formula.git_username:
+            continue
+
+        url_bits = urlsplit(formula.uri)
+
+        # rebuild the uri with the username in it
+        formula.uri = urljoin((
+            url_bits.scheme,
+            '{}:{}'.format(formula.git_username, url_bits.netloc),
+            url_bits.path,
+            url_bits.query,
+            url_bits.fragment,
+        ))
+
+        formula.save()
+
+
+def remove_user_from_uri(apps, schema_editor):
+    Formula = apps.get_model('formulas', 'Formula')
+
+    for formula in Formula.objects.all():
+        url_bits = urlsplit(formula.uri)
+
+        # don't do it if it's an ssh formula
+        if 'ssh' in url_bits.scheme:
+            continue
+
+        if url_bits.username:
+            formula.git_username = url_bits.username
+
+            if url_bits.port:
+                new_netloc = '{}:{}'.format(url_bits.hostname, url_bits.port)
+            else:
+                new_netloc = url_bits.hostname
+
+            formula.uri = urljoin((
+                url_bits.scheme,
+                new_netloc,
+                url_bits.path,
+                url_bits.query,
+                url_bits.fragment,
+            ))
+
+            formula.save()
+
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -27,7 +78,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='formula',
             name='status',
-            field=model_utils.fields.StatusField(default=b'error', max_length=100, no_check_for_status=True, verbose_name='status'),
+            field=model_utils.fields.StatusField(default='error', max_length=100, no_check_for_status=True, verbose_name='status'),
         ),
         migrations.AlterField(
             model_name='formulacomponent',
@@ -39,4 +90,18 @@ class Migration(migrations.Migration):
             name='modified',
             field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'),
         ),
+        migrations.RunPython(add_user_to_uri, remove_user_from_uri),
+        migrations.RemoveField(
+            model_name='formula',
+            name='access_token',
+        ),
+        migrations.RemoveField(
+            model_name='formula',
+            name='git_username',
+        ),
+        migrations.AddField(
+            model_name='formula',
+            name='ssh_private_key',
+            field=models.TextField(blank=True, verbose_name='SSH Private Key'),
+        ),
     ]

stackdio/api/formulas/migrations/0003_0_8_0_migrations.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-04-08 18:19
+from __future__ import unicode_literals
+
+import os
+import shutil
+from django.conf import settings
+from django.db import migrations
+from stackdio.api.formulas.tasks import import_formula
+
+
+def fix_formulas_forwards(apps, schema_editor):
+    Formula = apps.get_model('formulas', 'Formula')
+
+    formula_dir = os.path.join(settings.FILE_STORAGE_DIRECTORY, 'formulas')
+
+    for formula in Formula.objects.all():
+        # Delete the old dir
+        repo_name = os.path.splitext(os.path.split(formula.uri)[-1])[0]
+        dir_name = '{}-{}'.format(formula.id, repo_name)
+        shutil.rmtree(os.path.join(formula_dir, dir_name), ignore_errors=True)
+
+        # re-import
+        import_formula.si(formula.id)()
+
+
+def fix_formulas_reverse(apps, schema_editor):
+    Formula = apps.get_model('formulas', 'Formula')
+
+    formula_dir = os.path.join(settings.FILE_STORAGE_DIRECTORY, 'formulas')
+
+    for formula in Formula.objects.all():
+        repo_name = os.path.splitext(os.path.split(formula.uri)[-1])[0]
+        dir_name = '{}-{}'.format(formula.id, repo_name)
+
+        new_dir = os.path.join(formula_dir, dir_name)
+
+        # Copy the HEAD over
+        shutil.move(os.path.join(new_dir, 'checkouts', 'HEAD'), new_dir)
+
+        # Blow away the old checkouts
+        shutil.rmtree(os.path.join(new_dir, 'checkouts'))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('formulas', '0002_0_8_0_migrations'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_formulas_forwards, fix_formulas_reverse),
+    ]