diff --git a/minecode/management/indexing.py b/minecode/management/indexing.py index 593e53d7..2e435ba9 100644 --- a/minecode/management/indexing.py +++ b/minecode/management/indexing.py @@ -7,7 +7,7 @@ from minecode.management.commands import get_error_message import logging import sys -from minecode.model_utils import merge_or_create_resource +from minecode.model_utils import update_or_create_resource from packagedcode.utils import combine_expressions import traceback from minecode.models import ScannableURI @@ -37,7 +37,7 @@ def index_package_files(package, scan_data, reindex=False): try: logger.info(f'Indexing Resources and fingerprints related to {package.package_url} from scan data') for resource in scan_data.get('files', []): - r, _, _ = merge_or_create_resource(package, resource) + r, _, _ = update_or_create_resource(package, resource) path = r.path sha1 = r.sha1 if sha1: diff --git a/minecode/model_utils.py b/minecode/model_utils.py index f0c78b77..cf557488 100644 --- a/minecode/model_utils.py +++ b/minecode/model_utils.py @@ -1,3 +1,4 @@ +import copy import logging import sys @@ -29,6 +30,7 @@ DEFAULT_PIPELINES = ( 'scan_single_package', 'fingerprint_codebase', + 'collect_symbols', ) @@ -416,7 +418,7 @@ def merge_or_create_package(scanned_package, visit_level): return package, created, merged, map_error -def merge_or_create_resource(package, resource_data): +def update_or_create_resource(package, resource_data): """ Using Resource data from `resource_data`, create or update the corresponding purldb Resource from `package`. @@ -425,12 +427,18 @@ def merge_or_create_resource(package, resource_data): `resource`, as well as booleans representing whether the Resource was created or if the Resources scan field data was updated. """ - merged = False + updated = False created = False resource = None path = resource_data.get('path') + + extra_data = copy.deepcopy(resource_data.get('extra_data', {})) + extra_data.pop("directory_content", None) + extra_data.pop("directory_structure", None) + try: resource = Resource.objects.get(package=package, path=path) + updated = True except Resource.DoesNotExist: resource = Resource( package=package, @@ -450,7 +458,9 @@ def merge_or_create_resource(package, resource_data): is_archive=resource_data.get('is_archive'), is_media=resource_data.get('is_media'), is_key_file=resource_data.get('is_key_file'), + extra_data=extra_data, ) created = True _ = resource.set_scan_results(resource_data, save=True) - return resource, created, merged + resource.update_extra_data(extra_data) + return resource, created, updated diff --git a/minecode/tests/test_model_utils.py b/minecode/tests/test_model_utils.py index 908a8bc0..2ad10150 100644 --- a/minecode/tests/test_model_utils.py +++ b/minecode/tests/test_model_utils.py @@ -7,15 +7,17 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from datetime import timedelta import os -from django.utils import timezone +from django.test import TransactionTestCase +from packagedcode.maven import _parse from minecode.model_utils import merge_or_create_package -from minecode.utils_test import JsonBasedTesting, MiningTestCase +from minecode.model_utils import update_or_create_resource +from minecode.utils_test import JsonBasedTesting +from minecode.utils_test import MiningTestCase from packagedb.models import Package -from packagedcode.maven import _parse +from packagedb.models import Resource class ModelUtilsTestCase(MiningTestCase, JsonBasedTesting): @@ -95,3 +97,59 @@ def test_merge_or_create_package_merge_package(self): updated_fields = data['updated_fields'] expected_updated_fields_loc = self.get_test_loc('model_utils/expected_updated_fields.json') self.check_expected_results(updated_fields, expected_updated_fields_loc, regen=False) + + +class UpdateORCreateResourceTest(TransactionTestCase): + def setUp(self): + self.package = Package.objects.create(download_url="test-pkg.com") + self.resource_path = "root/test.c" + self.old_extra_data = { + "source_symbols": [ + "Old-symb1", + "Old-symb2", + ] + } + + self.new_extra_data = { + "source_symbols": [ + "New-symb1", + "New-symb2", + ] + } + + self.resource = Resource.objects.create( + package=self.package, + path=self.resource_path, + extra_data=self.old_extra_data, + ) + + def test_update_or_create_resource_update(self): + self.assertEqual(self.old_extra_data, self.resource.extra_data) + + update_or_create_resource( + self.package, + {"extra_data": self.new_extra_data, "path": self.resource_path}, + ) + self.resource.refresh_from_db() + + self.assertEqual(self.new_extra_data, self.resource.extra_data) + + def test_update_or_create_resource_create(self): + update_or_create_resource( + self.package, + { + "type": "file", + "name": "test_new", + "extension": ".c", + "is_binary": False, + "is_text": False, + "is_archive": False, + "is_media": False, + "is_key_file": False, + "extra_data": self.new_extra_data, + "path": "root/test_new.c", + }, + ) + + resource = Resource.objects.get(path="root/test_new.c") + self.assertEqual(self.new_extra_data, resource.extra_data)