Short-circuit task to avoid repeating same tasks again

gopython3/core/models.py

@@ -83,11 +83,14 @@ def get_status(self):
         return self.status
 
     def add_distribution(self, distribution):
-        """ Add distribution to job """
-        package, _ = Package.objects.get_or_create(name=distribution.name)
-        spec, _ = Spec.objects.get_or_create(package=package, version=distribution.version)
+        """ Add distribution to job
+
+            Returns tuple (package, spec)
+        """
+        package, package_created = Package.objects.get_or_create(name=distribution.name)
+        spec, spec_created = Spec.objects.get_or_create(package=package, version=distribution.version)
         self.specs.add(spec)
-        return spec, package
+        return package, package_created, spec, spec_created
 
     def __str__(self):
         return 'Job %s [%s]' % (self.pk, self.status)

gopython3/core/tasks.py

@@ -24,8 +24,11 @@ def process_requirement(req, job_id):
     distribution = PyPI.get_distribution(req)
 
     job = Job.objects.get(pk=job_id)
-    spec, package = job.add_distribution(distribution)
-    # TODO: if spec is already parsed, maybe we can serve cache
+    package, package_created, spec, spec_created = job.add_distribution(distribution)
+
+    # if spec is already parsed before, no need to do anything
+    if not spec_created:
+        return
 
     pypi = query_pypi.s(spec.pk)
     if req.specs:
@@ -34,9 +37,12 @@ def process_requirement(req, job_id):
         # Celery can not chain 2 groups, so it's a callback chain for now
         pypi = pypi | process_latest_spec.si(req.name, package.pk) | query_pypi.s()
 
-    # TODO: if package was parsed not long ago, maybe we can serve cache
-
     notify = notify_completed_spec.si(spec.pk)
+
+    # if package was parsed before, no need to query github or travis again
+    if not package_created:
+        return (pypi | notify).delay()
+
     return (pypi | github_travis.s(package.pk) | notify).delay()
 
 

gopython3/core/tests/test_functional.py

@@ -33,9 +33,8 @@ class TestHttp(TestCase):
 
     @attr('functional')
     def test_index(self):
-        """ Need imporvement, does not catch anything yet
-            FIXME: gopython3-6, need admin tests too
-        """
+        # Needs imporvement, does not catch anything yet
+        # FIXME: gopython3-6, need admin tests too
         response = self.client.get('/')
         self.assertTrue('Go Python 3!' in str(response.content))
         self.assertEqual(response.status_code, 200)

gopython3/core/tests/test_unit.py

@@ -54,16 +54,20 @@ class JobSepcTest(TestCase):
 
     def test_process_requirement(self):
         job = JobFactory()
-        spec, package = job.add_distribution(*fake_distributions('Django==1.5.4'))
+        package, package_created, spec, spec_created = job.add_distribution(*fake_distributions('Django==1.5.4'))
 
         self.assertQuerysetEqual(Package.objects.all(), ['Django (django)'], transform=str, ordered=False)
         self.assertQuerysetEqual(job.specs.all(), ['Django==1.5.4'], transform=str, ordered=False)
+        self.assertTrue(package_created)
+        self.assertTrue(spec_created)
 
     def test_does_not_create_duplicate_specs(self):
         spec = SpecFactory(version='0.2.19', package__name='lettuce', package__slug='lettuce')
         job = JobFactory()
-        same_spec, same_package = job.add_distribution(*fake_distributions('lettuce==0.2.19'))
+        same_package, package_created, same_spec, spec_created = job.add_distribution(*fake_distributions('lettuce==0.2.19'))
 
+        self.assertFalse(package_created)
+        self.assertFalse(spec_created)
         assert Spec.objects.count() == 1
         assert Package.objects.count() == 1
         assert job.specs.all().first().version == spec.version