Merge pull request #58 from colab/refactor-data-import

Refactor data import

README.rst

@@ -64,7 +64,7 @@ To run Colab with development server you will have to:
 
 .. code-block::
 
-  colab-init-config > /etc/colab/settings.yaml
+  colab-admin initconfig > /etc/colab/settings.py
   
 2- Edit the configuration file. Make sure you set everything you need including **database** credentials.
 
@@ -92,4 +92,4 @@ How to run the tests
 Follow the steps below:
 
 * Go to vagrant/colab/
-* run: ./runtests.sh
+* run: ./runtests.sh

colab/management/__init__.py

@@ -1,20 +0,0 @@
-
-import os
-
-from django.core.management import ManagementUtility
-
-from .initconfig import initconfig
-
-
-def execute_from_command_line(argv=None):
-    """
-    A simple method that runs a ManagementUtility.
-    """
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "colab.settings")
-
-    utility = ManagementUtility(argv)
-    utility.execute()
-
-
-def run_colab_config(argv=None):
-    initconfig()

colab/management/commands/celery.py

@@ -0,0 +1,26 @@
+from __future__ import absolute_import, unicode_literals
+
+from celery.bin import celery
+
+from colab.celery import app
+from colab.queue.command import CeleryCommand
+
+base = celery.CeleryCommand(app=app)
+
+
+# this is a reimplementation of the djcelery 'celery' command
+#   taken from Sentry
+class Command(CeleryCommand):
+    """The celery command."""
+    help = 'celery commands, see celery help'
+    options = (CeleryCommand.options
+               + base.get_options()
+               + base.preload_options)
+
+    def run_from_argv(self, argv):
+        argv = self.handle_default_options(argv)
+        if self.requires_system_checks:
+            self.validate()
+        base.execute_from_commandline(
+            ['{0[0]} {0[1]}'.format(argv)] + argv[2:],
+        )

colab/management/initconfig.py → colab/management/commands/initconfig.py

@@ -1,4 +1,5 @@
 
+from django.core.management.base import BaseCommand
 from django.utils.crypto import get_random_string
 
 
@@ -79,7 +80,7 @@
 # from colab.plugins.utils.menu import colab_url_factory
 #
 # name = 'colab.plugins.gitlab'
-# verbose_name = 'Gitlab Proxy'
+# verbose_name = 'Gitlab Plugin'
 #
 # upstream = 'localhost'
 # #middlewares = []
@@ -114,7 +115,10 @@
 """
 
 
-def initconfig():
-    chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
-    secret_key = get_random_string(50, chars)
-    print(CONFIG_TEMPLATE.format(secret_key=secret_key))
+class Command(BaseCommand):
+    help = 'Returns an example config file for Colab'
+
+    def handle(self, *args, **kwargs):
+        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
+        secret_key = get_random_string(50, chars)
+        print(CONFIG_TEMPLATE.format(secret_key=secret_key))

colab/plugins/apps.py

@@ -1,6 +1,7 @@
 
 from django.apps import AppConfig
 
+from .data import register_tasks
 from .utils.signals import connect_signal, register_signal
 
 
@@ -10,3 +11,5 @@ class PluginAppConfig(AppConfig):
     def ready(self):
         register_signal()
         connect_signal()
+
+        register_tasks()

colab/plugins/conf.py

@@ -0,0 +1,6 @@
+
+from django.conf import settings
+
+
+def get_plugin_config(app_label):
+    return settings.COLAB_APPS.get('gitlab', {})

colab/plugins/data/__init__.py

@@ -0,0 +1,3 @@
+
+from .base_importer import PluginDataImporter  # noqa
+from .tasks import TASKS, data_import, register_tasks  # noqa

colab/plugins/data/base_importer.py

@@ -0,0 +1,19 @@
+
+import abc
+
+from django.conf import settings
+
+
+class PluginDataImporter(object):
+
+    def __init__(self):
+        self.config = settings.COLAB_APPS.get(self.app_label, {})
+
+    @abc.abstractmethod
+    def fetch_data(self):
+        raise NotImplementedError
+    fetch_data.is_abstract = True
+
+    @abc.abstractmethod
+    def app_label(self):
+        raise NotImplementedError

colab/plugins/data/tasks.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import importlib
+import logging
+
+import redis
+
+from django.conf import settings
+
+from colab.celery import app
+
+LOGGER = logging.getLogger('colab.plugins.data')
+TASKS = set()
+
+
+def lock(method, name):
+    def wrapped_method(self, *args, **kwargs):
+        lock_id = 'colab-data-importer-{}'.format(name)
+        lock = redis.Redis().lock(lock_id)
+
+        if lock.acquire(blocking=False):
+            try:
+                return method(*args, **kwargs)
+            finally:
+                lock.release()
+
+    return wrapped_method
+
+
+def register_tasks():
+
+    global TASKS
+
+    for app_name in settings.INSTALLED_APPS:
+
+        module_name = '{}.data_importer'.format(app_name)
+        try:
+            module = importlib.import_module(module_name)
+        except ImportError:
+            continue
+
+        for item_name in dir(module):
+            item = getattr(module, item_name)
+
+            if callable(getattr(item, 'fetch_data', None)):
+                if getattr(item.fetch_data, 'is_abstract', False):
+                    continue
+                instance = item()
+                task_name = '{}.{}'.format(module.__name__, item_name)
+                thread_safe_method = lock(instance.fetch_data, task_name)
+                task = app.task(name=task_name, bind=True)(thread_safe_method)
+                TASKS.add(task)
+                LOGGER.debug('Registered task: %s', task_name)
+
+    LOGGER.debug(TASKS)
+    return TASKS
+
+
+def data_import(self):
+    for task in TASKS:
+        task.delay()

colab/plugins/gitlab/__init__.py

@@ -1,3 +1,3 @@
 
 
-default_app_config = 'colab.plugins.gitlab.apps.ProxyGitlabAppConfig'
+default_app_config = 'colab.plugins.gitlab.apps.GitlabPluginAppConfig'

colab/plugins/gitlab/apps.py

@@ -1,10 +1,10 @@
 
-from ..utils.apps import ColabProxiedAppConfig
+from ..utils.apps import ColabPluginAppConfig
 from colab.plugins.gitlab.tasks import handling_method
 from colab.signals.signals import register_signal, connect_signal
 
 
-class ProxyGitlabAppConfig(ColabProxiedAppConfig):
+class GitlabPluginAppConfig(ColabPluginAppConfig):
     name = 'colab.plugins.gitlab'
     verbose_name = 'Gitlab Plugin'
     short_name = 'gitlab'

colab/plugins/gitlab/data_api.py → colab/plugins/gitlab/data_importer.py

@@ -5,23 +5,22 @@
 
 from dateutil.parser import parse
 
-from django.conf import settings
 from django.db.models.fields import DateTimeField
+from colab.plugins.data import PluginDataImporter
+
+from .models import (GitlabProject, GitlabMergeRequest,
+                     GitlabComment, GitlabIssue)
 
-from colab.plugins.gitlab.models import (GitlabProject, GitlabMergeRequest,
-                                         GitlabComment, GitlabIssue)
-from colab.plugins.utils.proxy_data_api import ProxyDataAPI
 
 LOGGER = logging.getLogger('colab.plugin.gitlab')
 
 
-class GitlabDataAPI(ProxyDataAPI):
+class GitlabDataImporter(PluginDataImporter):
+    app_label = 'gitlab'
 
     def get_request_url(self, path, **kwargs):
-        proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
-
-        upstream = proxy_config.get('upstream')
-        kwargs['private_token'] = proxy_config.get('private_token')
+        upstream = self.config.get('upstream')
+        kwargs['private_token'] = self.config.get('private_token')
         params = urllib.urlencode(kwargs)
 
         if upstream[-1] == '/':
@@ -179,27 +178,40 @@ def fetch_comments_issues(self):
 
         return all_comments
 
+
+class GitlabProjectImporter(GitlabDataImporter):
+
     def fetch_data(self):
         LOGGER.info("Importing Projects")
         projects = self.fetch_projects()
         for datum in projects:
             datum.save()
 
+
+class GitlabMergeRequestImporter(GitlabDataImporter):
+
+    def fetch_data(self):
         LOGGER.info("Importing Merge Requests")
+        projects = GitlabProject.objects.all()
         merge_request_list = self.fetch_merge_request(projects)
         for datum in merge_request_list:
             datum.save()
 
+
+class GitlabIssueImporter(GitlabDataImporter):
+
+    def fetch_data(self):
         LOGGER.info("Importing Issues")
+        projects = GitlabProject.objects.all()
         issue_list = self.fetch_issue(projects)
         for datum in issue_list:
             datum.save()
 
+
+class GitlabCommentImporter(GitlabDataImporter):
+
+    def fetch_data(self):
         LOGGER.info("Importing Comments")
         comments_list = self.fetch_comments()
         for datum in comments_list:
             datum.save()
-
-    @property
-    def app_label(self):
-        return 'gitlab'

colab/plugins/gitlab/views.py

@@ -1,5 +1,5 @@
 
-from ..utils.views import ColabProxyView
+from colab.plugins.views import ColabProxyView
 
 
 class GitlabProxyView(ColabProxyView):

colab/plugins/mezuro/__init__.py

@@ -1,3 +1,3 @@
 
 
-default_app_config = 'colab.plugins.mezuro.apps.ProxyMezuroAppConfig'
+default_app_config = 'colab.plugins.mezuro.apps.MezuroPluginAppConfig'

colab/plugins/mezuro/apps.py

@@ -1,7 +1,7 @@
 
-from ..utils.apps import ColabProxiedAppConfig
+from ..utils.apps import ColabPluginAppConfig
 
 
-class ProxyMezuroAppConfig(ColabProxiedAppConfig):
+class MezuroPluginAppConfig(ColabPluginAppConfig):
     name = 'colab.plugins.mezuro'
-    verbose_name = 'Mezuro Proxy'
+    verbose_name = 'Mezuro Plugin'

colab/plugins/mezuro/views.py

@@ -1,4 +1,5 @@
-from ..utils.views import ColabProxyView
+
+from colab.plugins.views import ColabProxyView
 
 
 class MezuroProxyView(ColabProxyView):

colab/plugins/noosfero/__init__.py

@@ -1,3 +1,3 @@
 
 
-default_app_config = 'colab.plugins.noosfero.apps.ProxyNoosferoAppConfig'
+default_app_config = 'colab.plugins.noosfero.apps.NoosferoPluginAppConfig'

colab/plugins/noosfero/apps.py

@@ -1,7 +1,7 @@
 
-from ..utils.apps import ColabProxiedAppConfig
+from ..utils.apps import ColabPluginAppConfig
 
 
-class ProxyNoosferoAppConfig(ColabProxiedAppConfig):
+class NoosferoPluginAppConfig(ColabPluginAppConfig):
     name = 'colab.plugins.noosfero'
-    verbose_name = 'Noosfero Proxy'
+    verbose_name = 'Noosfero Plugin'