Add collect_symbols to the default pipelines and store symbols in resources

minecode/management/indexing.py

@@ -7,7 +7,7 @@
 from minecode.management.commands import get_error_message
 import logging
 import sys
-from minecode.model_utils import merge_or_create_resource
+from minecode.model_utils import update_or_create_resource
 from packagedcode.utils import combine_expressions
 import traceback
 from minecode.models import ScannableURI
@@ -37,7 +37,7 @@ def index_package_files(package, scan_data, reindex=False):
     try:
         logger.info(f'Indexing Resources and fingerprints related to {package.package_url} from scan data')
         for resource in scan_data.get('files', []):
-            r, _, _ = merge_or_create_resource(package, resource)
+            r, _, _ = update_or_create_resource(package, resource)
             path = r.path
             sha1 = r.sha1
             if sha1:

minecode/model_utils.py

@@ -1,3 +1,4 @@
+import copy
 import logging
 import sys
 
@@ -29,6 +30,7 @@
 DEFAULT_PIPELINES = (
     'scan_single_package',
     'fingerprint_codebase',
+    'collect_symbols',
 )
 
 
@@ -416,7 +418,7 @@ def merge_or_create_package(scanned_package, visit_level):
     return package, created, merged, map_error
 
 
-def merge_or_create_resource(package, resource_data):
+def update_or_create_resource(package, resource_data):
     """
     Using Resource data from `resource_data`, create or update the
     corresponding purldb Resource from `package`.
@@ -425,12 +427,18 @@ def merge_or_create_resource(package, resource_data):
     `resource`, as well as booleans representing whether the Resource was
     created or if the Resources scan field data was updated.
     """
-    merged = False
+    updated = False
     created = False
     resource = None
     path = resource_data.get('path')
+
+    extra_data = copy.deepcopy(resource_data.get('extra_data', {}))
+    extra_data.pop("directory_content", None)
+    extra_data.pop("directory_structure", None)
+
     try:
         resource = Resource.objects.get(package=package, path=path)
+        updated = True
     except Resource.DoesNotExist:
         resource = Resource(
             package=package,
@@ -450,7 +458,9 @@ def merge_or_create_resource(package, resource_data):
             is_archive=resource_data.get('is_archive'),
             is_media=resource_data.get('is_media'),
             is_key_file=resource_data.get('is_key_file'),
+            extra_data=extra_data,
         )
         created = True
     _ = resource.set_scan_results(resource_data, save=True)
-    return resource, created, merged
+    resource.update_extra_data(extra_data)
+    return resource, created, updated

minecode/tests/test_model_utils.py

@@ -7,15 +7,17 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-from datetime import timedelta
 import os
 
-from django.utils import timezone
+from django.test import TransactionTestCase
+from packagedcode.maven import _parse
 
 from minecode.model_utils import merge_or_create_package
-from minecode.utils_test import JsonBasedTesting, MiningTestCase
+from minecode.model_utils import update_or_create_resource
+from minecode.utils_test import JsonBasedTesting
+from minecode.utils_test import MiningTestCase
 from packagedb.models import Package
-from packagedcode.maven import _parse
+from packagedb.models import Resource
 
 
 class ModelUtilsTestCase(MiningTestCase, JsonBasedTesting):
@@ -95,3 +97,59 @@ def test_merge_or_create_package_merge_package(self):
         updated_fields = data['updated_fields']
         expected_updated_fields_loc = self.get_test_loc('model_utils/expected_updated_fields.json')
         self.check_expected_results(updated_fields, expected_updated_fields_loc, regen=False)
+
+
+class UpdateORCreateResourceTest(TransactionTestCase):
+    def setUp(self):
+        self.package = Package.objects.create(download_url="test-pkg.com")
+        self.resource_path = "root/test.c"
+        self.old_extra_data = {
+            "source_symbols": [
+                "Old-symb1",
+                "Old-symb2",
+            ]
+        }
+
+        self.new_extra_data = {
+            "source_symbols": [
+                "New-symb1",
+                "New-symb2",
+            ]
+        }
+
+        self.resource = Resource.objects.create(
+            package=self.package,
+            path=self.resource_path,
+            extra_data=self.old_extra_data,
+        )
+
+    def test_update_or_create_resource_update(self):
+        self.assertEqual(self.old_extra_data, self.resource.extra_data)
+
+        update_or_create_resource(
+            self.package,
+            {"extra_data": self.new_extra_data, "path": self.resource_path},
+        )
+        self.resource.refresh_from_db()
+
+        self.assertEqual(self.new_extra_data, self.resource.extra_data)
+
+    def test_update_or_create_resource_create(self):
+        update_or_create_resource(
+            self.package,
+            {
+                "type": "file",
+                "name": "test_new",
+                "extension": ".c",
+                "is_binary": False,
+                "is_text": False,
+                "is_archive": False,
+                "is_media": False,
+                "is_key_file": False,
+                "extra_data": self.new_extra_data,
+                "path": "root/test_new.c",
+            },
+        )
+
+        resource = Resource.objects.get(path="root/test_new.c")
+        self.assertEqual(self.new_extra_data, resource.extra_data)