139 api indexing multiple purls

packagedb/api.py

@@ -294,6 +294,55 @@ def get_enhanced_package_data(self, request, *args, **kwargs):
         package_data = get_enhanced_package(package)
         return Response(package_data)
 
+    @action(detail=False, methods=['post'])
+    def index_packages(self, request, *args, **kwargs):
+        """
+        Take a list of `package_urls` and add them to the package indexing
+        queue. Then return a mapping containing:
+
+        - queued_packages_count
+            - The number of package urls placed on the queue.
+        - queued_packages
+            - A list of package urls that were placed on the queue.
+        - unqueued_packages_count
+            - The number of package urls not placed on the queue. This is
+              because the package url already exists on the queue and has not
+              yet been processed.
+        - unqueued_packages
+            - A list of package urls that were not placed on the queue.
+        - unsupported_packages_count
+            - The number of package urls that are not processable by the queue.
+        - unsupported_packages
+            - A list of package urls that are not processable by the queue. The
+              package indexing queue can only handle npm and maven purls.
+        """
+        purls = request.data.getlist('package_urls')
+        unique_purls = set(purls)
+        queued_packages = []
+        unqueued_packages = []
+        unsupported_packages = []
+
+        for purl in unique_purls:
+            is_routable_purl = priority_router.is_routable(purl)
+            if not is_routable_purl:
+                unsupported_packages.append(purl)
+            else:
+                # add to queue
+                priority_resource_uri = PriorityResourceURI.objects.insert(purl)
+                if priority_resource_uri:
+                    queued_packages.append(purl)
+                else:
+                    unqueued_packages.append(purl)
+        response_data = {
+            'queued_packages_count': len(queued_packages),
+            'queued_packages': queued_packages,
+            'unqueued_packages_count': len(unqueued_packages),
+            'unqueued_packages': unqueued_packages,
+            'unsupported_packages_count': len(unsupported_packages),
+            'unsupported_packages': unsupported_packages,
+        }
+        return Response(response_data)
+
 
 UPDATEABLE_FIELDS = [
     'primary_language',

packagedb/tests/test_api.py

@@ -18,6 +18,7 @@
 from rest_framework import status
 from rest_framework.test import APIClient
 
+from minecode.models import PriorityResourceURI
 from minecode.utils_test import JsonBasedTesting
 from packagedb.models import Package
 from packagedb.models import PackageContentType
@@ -368,6 +369,66 @@ def test_api_package_list_endpoint_multiple_char_filters(self):
         self.assertIn(self.package3.purl, purls)
         self.assertNotIn(self.package.purl, purls)
 
+    def test_package_api_index_packages_endpoint(self):
+        priority_resource_uris_count = PriorityResourceURI.objects.all().count()
+        self.assertEqual(0, priority_resource_uris_count)
+        purls = [
+            'pkg:maven/ch.qos.reload4j/reload4j@1.2.24',
+            'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0',
+            'pkg:bitbucket/example/example@1.0.0',
+        ]
+        data = {
+            'package_urls': purls
+        }
+        response = self.client.post('/api/packages/index_packages/', data=data)
+        self.assertEqual(2, response.data['queued_packages_count'])
+        expected_queued_packages = [
+            'pkg:maven/ch.qos.reload4j/reload4j@1.2.24',
+            'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0',
+        ]
+        self.assertEqual(
+            sorted(expected_queued_packages),
+            sorted(response.data['queued_packages'])
+        )
+        self.assertEqual(0, response.data['unqueued_packages_count'])
+        self.assertEqual([], response.data['unqueued_packages'])
+        self.assertEqual(1, response.data['unsupported_packages_count'])
+        expected_unsupported_packages = [
+            'pkg:bitbucket/example/example@1.0.0'
+        ]
+        self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages'])
+        priority_resource_uris_count = PriorityResourceURI.objects.all().count()
+        self.assertEqual(2, priority_resource_uris_count)
+
+        # Ensure that we don't add the same packages to the queue if they have
+        # not yet been processed
+        purls = [
+            'pkg:maven/ch.qos.reload4j/reload4j@1.2.24',
+            'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0',
+            'pkg:bitbucket/example/example@1.0.0',
+        ]
+        data = {
+            'package_urls': purls
+        }
+        response = self.client.post('/api/packages/index_packages/', data=data)
+        self.assertEqual(0, response.data['queued_packages_count'])
+        self.assertEqual([], response.data['queued_packages'])
+        self.assertEqual(2, response.data['unqueued_packages_count'])
+        expected_unqueued_packages = [
+            'pkg:maven/ch.qos.reload4j/reload4j@1.2.24',
+            'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0',
+        ]
+        self.assertEqual(
+            sorted(expected_unqueued_packages),
+            sorted(response.data['unqueued_packages'])
+        )
+        self.assertEqual(1, response.data['unsupported_packages_count'])
+        expected_unsupported_packages = [
+            'pkg:bitbucket/example/example@1.0.0'
+        ]
+        self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages'])
+
+
 class PackageApiPurlFilterTestCase(JsonBasedTesting, TestCase):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles')