Properly handling 2nd pypi query

gopython3/api/pypi.py

@@ -34,13 +34,16 @@ def get_info(self, name, version=None):
     def get_distribution(cls, requirement):
         """ Resolve requirement
 
-            Receives a named tuple of parsed requirement:
+            Receives string or a named tuple of parsed requirement:
             Requirement(name='Django', specs=[('>=', '1.5'), ('<', '1.6')], extras=[])
         """
-        distlib_line = requirement.name
-        if requirement.specs:
-            # E.g.: Django (>= 1.0, < 2.0, != 1.3)
-            distlib_line += ' (%s)' % (', '.join('%s %s' % cond for cond in requirement.specs))
+        if isinstance(requirement, str):
+            distlib_line = requirement
+        else:
+            distlib_line = requirement.name
+            if requirement.specs:
+                # E.g.: Django (>= 1.0, < 2.0, != 1.3)
+                distlib_line += ' (%s)' % (', '.join('%s %s' % cond for cond in requirement.specs))
 
         # Returned object has canonical name (flask -> Flask)
         return locate(distlib_line)

gopython3/core/management/__init__.py

@@ -0,0 +1 @@
+# coding: utf-8

gopython3/core/management/commands/parse.py

@@ -0,0 +1,12 @@
+# coding: utf-8
+import logging
+from django.core.management.base import LabelCommand
+from core.models import Job
+
+
+class Command(LabelCommand):
+
+    def handle(self, *args, **options):
+        logger = logging.getLogger('core')
+        logger.setLevel(logging.DEBUG)
+        Job.objects.create_from_requirements('\n'.join(args))

gopython3/core/tasks.py

@@ -30,13 +30,13 @@ def process_requirement(req, job_id):
     pypi = query_pypi.s(spec.pk)
     if req.specs:
         # If version is not fixed, we already got latest package
-        # TODO: make sure, we save all data at once, to avoid inconsistency
-        process_latest_spec.s(req.name, package.pk) | query_pypi.s()
+        # Otherwise, fetch latest package to see if it is py3 compatible
+        # Celery can not chain 2 groups, so it's a callback chain for now
+        pypi = pypi | process_latest_spec.si(req.name, package.pk) | query_pypi.s()
 
     # TODO: if package was parsed not long ago, maybe we can serve cache
 
     notify = notify_completed_spec.si(spec.pk)
-
     return (pypi | github_travis.s(package.pk) | notify).delay()
 
 
@@ -47,17 +47,23 @@ def process_latest_spec(package_name, package_id):
         Obtaining only metadata from PyPI, because other tasks will query latest repo version anyway.
     """
     from .models import Spec
+    logger.debug('Creating latest spec')
 
     distribution = PyPI.get_distribution(package_name)
-    latest_spec, _ = Spec.objects.get_or_create(package_id=package_id, version=distribution.version)
-    return latest_spec.pk
-
+    latest_spec, created = Spec.objects.get_or_create(package_id=package_id, version=distribution.version)
+    if created:
+        # do not return spec, because it's already queued
+        return latest_spec.pk
 
 @task
 def query_pypi(spec_pk):
     """ Query one spec of package on PyPI"""
     from .models import Spec
 
+    # Short-circuit, if we have no spec to query
+    if not spec_pk:
+        return
+
     spec = Spec.objects.get(pk=spec_pk)
     logger.debug('[PYPI] Fetching data for %s' % spec)
     pkg_data = PyPI().get_info(spec.name, spec.version)