From 16a5725a4e000078c8299331e01d5c6e4e29006c Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 8 Jun 2026 10:13:03 +0400 Subject: [PATCH 1/4] feat: add create_dependencies option to all import forms Signed-off-by: tdruez --- .gitignore | 1 + product_portfolio/api.py | 20 +++++++++++ product_portfolio/forms.py | 36 +++++++++++++++++++ product_portfolio/importers.py | 32 ++++++++++++----- .../0017_scancodeproject_import_options.py | 18 ++++++++++ product_portfolio/models.py | 9 +++++ 6 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 product_portfolio/migrations/0017_scancodeproject_import_options.py diff --git a/.gitignore b/.gitignore index 128f6ab5..5b72812d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ include .settings TAGS .idea +.vscode Include Lib .env diff --git a/product_portfolio/api.py b/product_portfolio/api.py index d24c5076..a84d3439 100644 --- a/product_portfolio/api.py +++ b/product_portfolio/api.py @@ -243,6 +243,11 @@ class LoadSBOMsFormSerializer(serializers.Serializer): default=False, help_text=LoadSBOMsForm.base_fields["scan_all_packages"].help_text, ) + create_dependencies = serializers.BooleanField( + required=False, + default=False, + help_text=LoadSBOMsForm.base_fields["create_dependencies"].help_text, + ) class ImportManifestsFormSerializer(serializers.Serializer): @@ -268,6 +273,11 @@ class ImportManifestsFormSerializer(serializers.Serializer): default=False, help_text=ImportManifestsForm.base_fields["scan_all_packages"].help_text, ) + create_dependencies = serializers.BooleanField( + required=False, + default=False, + help_text=ImportManifestsForm.base_fields["create_dependencies"].help_text, + ) class ImportFromScanSerializer(serializers.Serializer): @@ -281,6 +291,11 @@ class ImportFromScanSerializer(serializers.Serializer): default=False, help_text=ImportFromScanForm.base_fields["create_codebase_resources"].help_text, ) + create_dependencies = serializers.BooleanField( + required=False, + default=False, + help_text=ImportFromScanForm.base_fields["create_dependencies"].help_text, + ) stop_on_error = serializers.BooleanField( required=False, default=False, @@ -300,6 +315,11 @@ class PullProjectDataSerializer(serializers.Serializer): default=False, help_text=PullProjectDataForm.base_fields["update_existing_packages"].help_text, ) + create_dependencies = serializers.BooleanField( + required=False, + default=False, + help_text=PullProjectDataForm.base_fields["create_dependencies"].help_text, + ) class ScanCodeProjectSerializer(DataspacedSerializer): diff --git a/product_portfolio/forms.py b/product_portfolio/forms.py index f6ce25d4..7119230d 100644 --- a/product_portfolio/forms.py +++ b/product_portfolio/forms.py @@ -554,6 +554,15 @@ class ImportFromScanForm(forms.Form): "imported Packages." ), ) + create_dependencies = forms.BooleanField( + label=_("Create Dependencies"), + required=False, + initial=False, + help_text=_( + "When checked, dependency relationships between packages discovered in the " + "import will be created on the Product." + ), + ) stop_on_error = forms.BooleanField( label=_("Stop and cancel import on data validation error"), required=False, @@ -580,6 +589,7 @@ def helper(self): None, "upload_file", "create_codebase_resources", + "create_dependencies", "stop_on_error", StrictSubmit("submit", _("Import"), css_class="btn-success col-2"), ), @@ -595,6 +605,7 @@ def save(self, product): self.user, upload_file=self.cleaned_data.get("upload_file"), create_codebase_resources=self.cleaned_data.get("create_codebase_resources"), + create_dependencies=self.cleaned_data.get("create_dependencies"), stop_on_error=self.cleaned_data.get("stop_on_error"), ) @@ -650,6 +661,15 @@ class BaseProductImportFormView(forms.Form): "from the Package URL (purl). A download URL is required for package scanning." ), ) + create_dependencies = forms.BooleanField( + label=_("Create Dependencies"), + required=False, + initial=False, + help_text=_( + "When checked, dependency relationships between packages discovered in the " + "import will be created on the Product." + ), + ) @property def helper(self): @@ -664,6 +684,7 @@ def helper(self): "infer_download_urls", "update_existing_packages", "scan_all_packages", + "create_dependencies", StrictSubmit("submit", _("Import"), css_class="btn-success col-2"), ), ) @@ -678,6 +699,9 @@ def submit(self, product, user): update_existing_packages=self.cleaned_data.get("update_existing_packages"), scan_all_packages=self.cleaned_data.get("scan_all_packages"), infer_download_urls=self.cleaned_data.get("infer_download_urls"), + import_options={ + "create_dependencies": self.cleaned_data.get("create_dependencies", False), + }, created_by=user, ) @@ -975,6 +999,15 @@ class PullProjectDataForm(forms.Form): "without any modification." ), ) + create_dependencies = forms.BooleanField( + label=_("Create Dependencies"), + required=False, + initial=False, + help_text=_( + "When checked, dependency relationships between packages discovered in the " + "import will be created on the Product." + ), + ) @property def helper(self): @@ -1007,6 +1040,9 @@ def submit(self, product, user): project_uuid=project_data.get("uuid"), update_existing_packages=self.cleaned_data.get("update_existing_packages"), scan_all_packages=False, + import_options={ + "create_dependencies": self.cleaned_data.get("create_dependencies", False), + }, status=ScanCodeProject.Status.SUBMITTED, created_by=user, ) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 84c9ac0c..76006b6d 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -383,13 +383,20 @@ def save_all(self): class ImportFromScan: def __init__( - self, product, user, upload_file, create_codebase_resources=True, stop_on_error=False + self, + product, + user, + upload_file, + create_codebase_resources=True, + create_dependencies=False, + stop_on_error=False, ): self.product = product self.dataspace = product.dataspace self.user = user self.upload_file = upload_file self.create_codebase_resources = create_codebase_resources + self.create_dependencies = create_dependencies self.stop_on_error = stop_on_error self.data = {} @@ -475,17 +482,21 @@ def import_packages(self): '"packages" is empty in the uploaded json file.' ) - dependencies = self.data.get("dependencies", []) dependencies_by_package_uid = defaultdict(list) - for dependency in dependencies: - for_package_uid = dependency.get("for_package_uid") - dependencies_by_package_uid[for_package_uid].append(dependency) + if self.create_dependencies: + dependencies = self.data.get("dependencies", []) + for dependency in dependencies: + for_package_uid = dependency.get("for_package_uid") + dependencies_by_package_uid[for_package_uid].append(dependency) for package_data in packages: package_uid = package_data.get("package_uid") - package_dependencies = package_data.get("dependencies", []) - if not package_dependencies: - package_data["dependencies"] = dependencies_by_package_uid.get(package_uid, []) + if self.create_dependencies: + package_dependencies = package_data.get("dependencies", []) + if not package_dependencies: + package_data["dependencies"] = dependencies_by_package_uid.get(package_uid, []) + else: + package_data.pop("dependencies", None) prepared = PackageImporter.prepare_package(package_data, path="/") if not prepared: @@ -658,6 +669,7 @@ def __init__( update_existing=False, scan_all_packages=False, infer_download_urls=False, + create_dependencies=False, ): self.licensing = Licensing() self.created = defaultdict(list) @@ -672,6 +684,7 @@ def __init__( self.update_existing = update_existing self.scan_all_packages = scan_all_packages self.infer_download_urls = infer_download_urls + self.create_dependencies = create_dependencies scancodeio = ScanCodeIO(user.dataspace) self.packages = scancodeio.fetch_project_packages(self.project_uuid) @@ -681,7 +694,8 @@ def __init__( def save(self): self.import_packages() - self.import_dependencies() + if self.create_dependencies: + self.import_dependencies() if self.scan_all_packages: transaction.on_commit(lambda: self.product.scan_all_packages_task(self.user)) diff --git a/product_portfolio/migrations/0017_scancodeproject_import_options.py b/product_portfolio/migrations/0017_scancodeproject_import_options.py new file mode 100644 index 00000000..33fbb65d --- /dev/null +++ b/product_portfolio/migrations/0017_scancodeproject_import_options.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.5 on 2026-06-08 05:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('product_portfolio', '0016_alter_productcomponent_weighted_risk_score_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='scancodeproject', + name='import_options', + field=models.JSONField(blank=True, default=dict, help_text='A dictionary of options used to configure the import process. New options can be added here without requiring a database migration.'), + ), + ] diff --git a/product_portfolio/models.py b/product_portfolio/models.py index 90fdac71..9fb8e6c9 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -1712,6 +1712,14 @@ class Status(models.TextChoices): infer_download_urls = models.BooleanField( default=False, ) + import_options = models.JSONField( + blank=True, + default=dict, + help_text=_( + "A dictionary of options used to configure the import process. " + "New options can be added here without requiring a database migration." + ), + ) status = models.CharField( max_length=10, choices=Status.choices, @@ -1773,6 +1781,7 @@ def import_data_from_scancodeio(self): update_existing=self.update_existing_packages, scan_all_packages=self.scan_all_packages, infer_download_urls=self.infer_download_urls, + create_dependencies=self.import_options.get("create_dependencies", False), ) created, existing, errors = importer.save() From 6e0787da071794b2d6fb1404ff9d78a2527459b8 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 8 Jun 2026 12:44:43 +0400 Subject: [PATCH 2/4] add unit tests Signed-off-by: tdruez --- product_portfolio/importers.py | 14 ++-- product_portfolio/tests/test_importers.py | 81 +++++++++++++++++++ .../import_from_scan_with_dependencies.json | 37 +++++++++ 3 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 product_portfolio/tests/testfiles/import_from_scan_with_dependencies.json diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 76006b6d..2fdcda92 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -472,6 +472,13 @@ def validate_toolkit_options(scan_options): options_str = " ".join(missing_options) raise ValidationError(f"The Scan run is missing those required options: {options_str}") + def _handle_package_dependencies(self, package_data, package_uid, dependencies_by_package_uid): + if self.create_dependencies: + if not package_data.get("dependencies"): + package_data["dependencies"] = dependencies_by_package_uid.get(package_uid, []) + else: + package_data.pop("dependencies", None) + def import_packages(self): product_packages_count = 0 packages_count = 0 @@ -491,12 +498,7 @@ def import_packages(self): for package_data in packages: package_uid = package_data.get("package_uid") - if self.create_dependencies: - package_dependencies = package_data.get("dependencies", []) - if not package_dependencies: - package_data["dependencies"] = dependencies_by_package_uid.get(package_uid, []) - else: - package_data.pop("dependencies", None) + self._handle_package_dependencies(package_data, package_uid, dependencies_by_package_uid) prepared = PackageImporter.prepare_package(package_data, path="/") if not prepared: diff --git a/product_portfolio/tests/test_importers.py b/product_portfolio/tests/test_importers.py index cdfe83b5..85b1c2ee 100644 --- a/product_portfolio/tests/test_importers.py +++ b/product_portfolio/tests/test_importers.py @@ -6,6 +6,7 @@ # See https://aboutcode.org for more information about AboutCode FOSS projects. # +import json import tempfile import uuid from pathlib import Path @@ -908,6 +909,36 @@ def test_product_portfolio_product_import_from_scan_input_data_validation_errors self.assertEqual(expected, warnings) self.assertEqual({"Packages": 1, "Product Packages": 1}, created_counts) + def test_product_portfolio_product_import_from_scan_create_dependencies(self): + scan_input_location = self.testfiles_path / "import_from_scan_with_dependencies.json" + + upload_file = wrap_as_temp_uploaded_file(scan_input_location) + importer = ImportFromScan( + self.product1, self.super_user, upload_file, create_dependencies=False + ) + importer.save() + package = self.product1.packages.get() + self.assertEqual([], package.dependencies) + + package.productpackages.all().delete() + package.delete() + + upload_file = wrap_as_temp_uploaded_file(scan_input_location) + importer = ImportFromScan( + self.product1, self.super_user, upload_file, create_dependencies=True + ) + importer.save() + package = self.product1.packages.get() + expected_dependency = { + "purl": "pkg:npm/lodash@4.17.21", + "scope": "runtime", + "is_runtime": True, + "is_optional": False, + "is_pinned": True, + "for_package_uid": "pkg:npm/test-package@1?uuid=9779a0ea-ef30-4a05-b4db-0a0ba3b3507c", + } + self.assertEqual([expected_dependency], json.loads(package.dependencies)) + def test_product_portfolio_product_import_from_scan_view_base(self): self.client.login(username=self.super_user.username, password="secret") scan_input_location = self.testfiles_path / "import_from_scan.json" @@ -1060,6 +1091,56 @@ def test_product_portfolio_import_packages_from_scancodeio_importer( importer.save() mock_fetch.assert_called() + @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_project_dependencies") + @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_project_packages") + def test_product_portfolio_import_packages_from_scancodeio_create_dependencies( + self, mock_fetch_packages, mock_fetch_dependencies + ): + purl = "pkg:maven/abc/abc@1.0" + mock_fetch_packages.return_value = [ + { + "type": "maven", + "namespace": "abc", + "name": "abc", + "version": "1.0", + "purl": purl, + } + ] + dependency_uid = "pkg:pypi/requests@2.0?uuid=test-dep" + mock_fetch_dependencies.return_value = [ + { + "purl": "pkg:pypi/requests@2.0", + "dependency_uid": dependency_uid, + "for_package_uid": None, + "resolved_to_package_uid": None, + "scope": "install", + "is_runtime": True, + "is_optional": False, + "is_pinned": False, + "is_direct": True, + "datasource_id": "pypi_setup_cfg", + "affected_by_vulnerabilities": [], + } + ] + + importer = ImportPackageFromScanCodeIO( + user=self.super_user, + project_uuid=uuid.uuid4(), + product=self.product1, + create_dependencies=False, + ) + importer.save() + self.assertEqual(0, self.product1.dependencies.count()) + + importer = ImportPackageFromScanCodeIO( + user=self.super_user, + project_uuid=uuid.uuid4(), + product=self.product1, + create_dependencies=True, + ) + importer.save() + self.assertEqual(1, self.product1.dependencies.count()) + @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_project_dependencies") @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_project_packages") def test_product_portfolio_import_packages_from_scio_importer_multiple_package_objs( diff --git a/product_portfolio/tests/testfiles/import_from_scan_with_dependencies.json b/product_portfolio/tests/testfiles/import_from_scan_with_dependencies.json new file mode 100644 index 00000000..acd2df41 --- /dev/null +++ b/product_portfolio/tests/testfiles/import_from_scan_with_dependencies.json @@ -0,0 +1,37 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "32.0.0", + "options": { + "--copyright": true, + "--info": true, + "--license": true, + "--package": true + } + } + ], + "packages": [ + { + "type": "npm", + "name": "test-package", + "version": "1", + "purl": "pkg:npm/test-package@1", + "package_uid": "pkg:npm/test-package@1?uuid=9779a0ea-ef30-4a05-b4db-0a0ba3b3507c", + "download_url": "https://registry.npmjs.org/test-package/-/test-package-1.tgz", + "copyright": "Copyright", + "declared_license_expression": "apache-2.0" + } + ], + "dependencies": [ + { + "purl": "pkg:npm/lodash@4.17.21", + "scope": "runtime", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "for_package_uid": "pkg:npm/test-package@1?uuid=9779a0ea-ef30-4a05-b4db-0a0ba3b3507c" + } + ], + "files": [] +} From a23a4a751d558a56ded0f1529e5611e7ae12ad58 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 8 Jun 2026 13:09:11 +0400 Subject: [PATCH 3/4] fix code format Signed-off-by: tdruez --- product_portfolio/importers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 2fdcda92..28d5768e 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -498,7 +498,9 @@ def import_packages(self): for package_data in packages: package_uid = package_data.get("package_uid") - self._handle_package_dependencies(package_data, package_uid, dependencies_by_package_uid) + self._handle_package_dependencies( + package_data, package_uid, dependencies_by_package_uid + ) prepared = PackageImporter.prepare_package(package_data, path="/") if not prepared: From 228b70d29e8ad5fb48a0ce10fc61fcdff2ed9df4 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 8 Jun 2026 13:31:59 +0400 Subject: [PATCH 4/4] fix failing tests Signed-off-by: tdruez --- product_portfolio/tests/test_importers.py | 5 +++++ product_portfolio/tests/test_views.py | 1 + 2 files changed, 6 insertions(+) diff --git a/product_portfolio/tests/test_importers.py b/product_portfolio/tests/test_importers.py index 85b1c2ee..f6cbe077 100644 --- a/product_portfolio/tests/test_importers.py +++ b/product_portfolio/tests/test_importers.py @@ -1043,6 +1043,7 @@ def test_product_portfolio_import_packages_from_scancodeio_importer( project_uuid=uuid.uuid4(), product=self.product1, infer_download_urls=True, + create_dependencies=True, ) created, existing, errors = importer.save() created_package_package_url = created.get("package")[0] @@ -1072,6 +1073,7 @@ def test_product_portfolio_import_packages_from_scancodeio_importer( user=self.super_user, project_uuid=uuid.uuid4(), product=self.product1, + create_dependencies=True, ) created, existing, errors = importer.save() self.assertEqual({}, created) @@ -1287,6 +1289,7 @@ def test_product_portfolio_import_packages_from_scio_importer_duplicate_dependen user=self.super_user, project_uuid=uuid.uuid4(), product=self.product1, + create_dependencies=True, ) created, existing, errors = importer.save() expected = { @@ -1304,6 +1307,7 @@ def test_product_portfolio_import_packages_from_scio_importer_duplicate_dependen user=self.super_user, project_uuid=uuid.uuid4(), product=self.product1, + create_dependencies=True, ) created, existing, errors = importer.save() self.assertEqual({}, created) @@ -1325,6 +1329,7 @@ def test_product_portfolio_import_packages_from_scio_importer_duplicate_dependen user=self.super_user, project_uuid=uuid.uuid4(), product=self.product1, + create_dependencies=True, ) created, existing, errors = importer.save() self.assertEqual({}, created) diff --git a/product_portfolio/tests/test_views.py b/product_portfolio/tests/test_views.py index 57755918..c4eb4ba2 100644 --- a/product_portfolio/tests/test_views.py +++ b/product_portfolio/tests/test_views.py @@ -3214,6 +3214,7 @@ def test_product_portfolio_import_packages_from_scancodeio_view( dataspace=self.product1.dataspace, type=ScanCodeProject.ProjectType.LOAD_SBOMS, created_by=self.super_user, + import_options={"create_dependencies": True}, ) view_name = "product_portfolio:import_packages_from_scancodeio"