From f03e7b7fee3160e6d218d43e8fd801def0f5c383 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:09:43 +0400 Subject: [PATCH] Add REST API endpoint to download ABOUT files and SPDX document #60 (#82) Signed-off-by: tdruez --- CHANGELOG.rst | 3 + dejacode/__init__.py | 4 + dje/outputs.py | 137 ++++++++++++++++++++++++++ dje/tests/test_outputs.py | 85 ++++++++++++++++ dje/views.py | 127 +++--------------------- docs/faq.rst | 17 ++++ docs/howto-3.rst | 26 +++++ docs/index.rst | 2 + product_portfolio/api.py | 34 ++++++- product_portfolio/tests/test_api.py | 51 ++++++++++ product_portfolio/tests/test_views.py | 26 +---- 11 files changed, 370 insertions(+), 142 deletions(-) create mode 100644 dje/outputs.py create mode 100644 dje/tests/test_outputs.py create mode 100644 docs/faq.rst create mode 100644 docs/howto-3.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bbcbd7f..b1ec3cc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,6 +28,9 @@ Release notes "Pull ScanCode.io project data" feature as Product action in the REST API. https://github.com/nexB/dejacode/issues/59 +- Add REST API endpoints to download SBOMs as CycloneDX and SPDX. + https://github.com/nexB/dejacode/issues/60 + - Refactor the "Import manifest" feature as "Load SBOMs". https://github.com/nexB/dejacode/issues/61 diff --git a/dejacode/__init__.py b/dejacode/__init__.py index 1bf536a..1248110 100644 --- a/dejacode/__init__.py +++ b/dejacode/__init__.py @@ -8,10 +8,14 @@ import os import sys +import warnings VERSION = "5.1.0-dev" __version__ = VERSION +# Turn off the warnings for the following modules. +warnings.filterwarnings("ignore", module="cyclonedx") + def command_line(): """Command line entry point.""" diff --git a/dje/outputs.py b/dje/outputs.py new file mode 100644 index 0000000..8dbc24d --- /dev/null +++ b/dje/outputs.py @@ -0,0 +1,137 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# DejaCode is a trademark of nexB Inc. +# SPDX-License-Identifier: AGPL-3.0-only +# See https://github.com/nexB/dejacode for support or download. +# See https://aboutcode.org for more information about AboutCode FOSS projects. +# + +import re + +from django.http import FileResponse +from django.http import Http404 + +from cyclonedx import output as cyclonedx_output +from cyclonedx.model import bom as cyclonedx_bom + +from dejacode import __version__ as dejacode_version +from dejacode_toolkit import spdx + + +def safe_filename(filename): + """Convert the provided `filename` to a safe filename.""" + return re.sub("[^A-Za-z0-9.-]+", "_", filename).lower() + + +def get_attachment_response(file_content, filename, content_type): + if not file_content or not filename: + raise Http404 + + response = FileResponse( + file_content, + filename=filename, + content_type=content_type, + ) + response["Content-Disposition"] = f'attachment; filename="{filename}"' + + return response + + +def get_spdx_extracted_licenses(spdx_packages): + """ + Return all the licenses to be included in the SPDX extracted_licenses. + Those include the `LicenseRef-` licenses, ie: licenses not available in the + SPDX list. + + In the case of Product relationships, ProductComponent and ProductPackage, + the set of licenses of the related object, Component or Package, is used + as the licenses of the relationship is always a subset of the ones of the + related object. + This ensures that we have all the license required for a valid SPDX document. + """ + from product_portfolio.models import ProductRelationshipMixin + + all_licenses = set() + for entry in spdx_packages: + if isinstance(entry, ProductRelationshipMixin): + all_licenses.update(entry.related_component_or_package.licenses.all()) + else: + all_licenses.update(entry.licenses.all()) + + return [ + license.as_spdx() for license in all_licenses if license.spdx_id.startswith("LicenseRef") + ] + + +def get_spdx_document(instance, user): + spdx_packages = instance.get_spdx_packages() + + creation_info = spdx.CreationInfo( + person_name=f"{user.first_name} {user.last_name}", + person_email=user.email, + organization_name=user.dataspace.name, + tool=f"DejaCode-{dejacode_version}", + ) + + document = spdx.Document( + name=f"dejacode_{instance.dataspace.name}_{instance._meta.model_name}_{instance}", + namespace=f"https://dejacode.com/spdxdocs/{instance.uuid}", + creation_info=creation_info, + packages=[package.as_spdx() for package in spdx_packages], + extracted_licenses=get_spdx_extracted_licenses(spdx_packages), + ) + + return document + + +def get_spdx_filename(spdx_document): + document_name = spdx_document.as_dict()["name"] + filename = f"{document_name}.spdx.json" + return safe_filename(filename) + + +def get_cyclonedx_bom(instance, user): + """https://cyclonedx.org/use-cases/#dependency-graph""" + cyclonedx_components = [] + + if hasattr(instance, "get_cyclonedx_components"): + cyclonedx_components = [ + component.as_cyclonedx() for component in instance.get_cyclonedx_components() + ] + + bom = cyclonedx_bom.Bom(components=cyclonedx_components) + + cdx_component = instance.as_cyclonedx() + cdx_component.dependencies.update([component.bom_ref for component in cyclonedx_components]) + + bom.metadata = cyclonedx_bom.BomMetaData( + component=cdx_component, + tools=[ + cyclonedx_bom.Tool( + vendor="nexB", + name="DejaCode", + version=dejacode_version, + ) + ], + authors=[ + cyclonedx_bom.OrganizationalContact( + name=f"{user.first_name} {user.last_name}", + ) + ], + ) + + return bom + + +def get_cyclonedx_bom_json(cyclonedx_bom): + outputter = cyclonedx_output.get_instance( + bom=cyclonedx_bom, + output_format=cyclonedx_output.OutputFormat.JSON, + ) + return outputter.output_as_string() + + +def get_cyclonedx_filename(instance): + base_filename = f"dejacode_{instance.dataspace.name}_{instance._meta.model_name}" + filename = f"{base_filename}_{instance}.cdx.json" + return safe_filename(filename) diff --git a/dje/tests/test_outputs.py b/dje/tests/test_outputs.py new file mode 100644 index 0000000..6194547 --- /dev/null +++ b/dje/tests/test_outputs.py @@ -0,0 +1,85 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# DejaCode is a trademark of nexB Inc. +# SPDX-License-Identifier: AGPL-3.0-only +# See https://github.com/nexB/dejacode for support or download. +# See https://aboutcode.org for more information about AboutCode FOSS projects. +# + +from django.test import TestCase + +from cyclonedx.model import bom as cyclonedx_bom + +from dejacode import __version__ as dejacode_version +from dje import outputs +from dje.models import Dataspace +from dje.tests import create_superuser +from dje.tests import create_user +from product_portfolio.models import Product + + +class OutputsTestCase(TestCase): + def setUp(self): + self.dataspace = Dataspace.objects.create(name="nexB") + self.super_user = create_superuser("nexb_user", self.dataspace) + self.basic_user = create_user("basic_user", self.dataspace) + + self.product1 = Product.objects.create( + name="Product1 With Space", version="1.0", dataspace=self.dataspace + ) + + def test_outputs_safe_filename(self): + self.assertEqual("low_-_up_", outputs.safe_filename("low -_UP*&//")) + + def test_outputs_get_attachment_response(self): + response = outputs.get_attachment_response( + file_content="AAA", filename="file.txt", content_type="application/json" + ) + expected = 'attachment; filename="file.txt"' + self.assertEqual(expected, response["Content-Disposition"]) + self.assertEqual("application/json", response["Content-Type"]) + + def test_outputs_get_spdx_document(self): + document = outputs.get_spdx_document(self.product1, self.super_user) + document.creation_info.created = "2000-01-01T01:02:03Z" + expected = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "dejacode_nexb_product_product1_with_space_1.0", + "documentNamespace": f"https://dejacode.com/spdxdocs/{self.product1.uuid}", + "creationInfo": { + "created": "2000-01-01T01:02:03Z", + "creators": [ + "Person: (user@email.com)", + "Organization: nexB ()", + f"Tool: DejaCode-{dejacode_version}", + ], + "licenseListVersion": "3.18", + }, + "packages": [], + "documentDescribes": [], + } + self.assertEqual(expected, document.as_dict()) + + def test_outputs_get_spdx_filename(self): + document = outputs.get_spdx_document(self.product1, self.super_user) + self.assertEqual( + "dejacode_nexb_product_product1_with_space_1.0.spdx.json", + outputs.get_spdx_filename(document), + ) + + def test_outputs_get_cyclonedx_bom(self): + bom = outputs.get_cyclonedx_bom(instance=self.product1, user=self.super_user) + self.assertIsInstance(bom, cyclonedx_bom.Bom) + + def test_outputs_get_cyclonedx_bom_json(self): + bom = outputs.get_cyclonedx_bom(instance=self.product1, user=self.super_user) + bom_json = outputs.get_cyclonedx_bom_json(bom) + self.assertTrue(bom_json.startswith('{"$schema":')) + + def test_outputs_get_cyclonedx_filename(self): + self.assertEqual( + "dejacode_nexb_product_product1_with_space_1.0.cdx.json", + outputs.get_cyclonedx_filename(instance=self.product1), + ) diff --git a/dje/views.py b/dje/views.py index 1fc38c7..0a80ab1 100644 --- a/dje/views.py +++ b/dje/views.py @@ -67,19 +67,16 @@ from django.views.generic.edit import DeleteView import django_otp -from cyclonedx import output as cyclonedx_output -from cyclonedx.model import bom as cyclonedx_bom from django_filters.views import FilterView from grappelli.views.related import AutocompleteLookup from grappelli.views.related import RelatedLookup from notifications import views as notifications_views from component_catalog.license_expression_dje import get_license_objects -from dejacode import __version__ as dejacode_version -from dejacode_toolkit import spdx from dejacode_toolkit.purldb import PurlDB from dejacode_toolkit.scancodeio import ScanCodeIO from dejacode_toolkit.vulnerablecode import VulnerableCode +from dje import outputs from dje.copier import COPY_DEFAULT_EXCLUDE from dje.copier import SKIP from dje.copier import get_object_in @@ -118,7 +115,6 @@ from dje.utils import group_by_simple from dje.utils import has_permission from dje.utils import queryset_to_changelist_href -from dje.utils import safe_filename from dje.utils import str_to_id_list License = apps.get_model("license_library", "License") @@ -2316,32 +2312,6 @@ def get_context_data(self, **kwargs): return context -def get_spdx_extracted_licenses(spdx_packages): - """ - Return all the licenses to be included in the SPDX extracted_licenses. - Those include the `LicenseRef-` licenses, ie: licenses not available in the - SPDX list. - - In the case of Product relationships, ProductComponent and ProductPackage, - the set of licenses of the related object, Component or Package, is used - as the licenses of the relationship is always a subset of the ones of the - related object. - This ensures that we have all the license required for a valid SPDX document. - """ - from product_portfolio.models import ProductRelationshipMixin - - all_licenses = set() - for entry in spdx_packages: - if isinstance(entry, ProductRelationshipMixin): - all_licenses.update(entry.related_component_or_package.licenses.all()) - else: - all_licenses.update(entry.licenses.all()) - - return [ - license.as_spdx() for license in all_licenses if license.spdx_id.startswith("LicenseRef") - ] - - class ExportSPDXDocumentView( LoginRequiredMixin, DataspaceScopeMixin, @@ -2349,43 +2319,14 @@ class ExportSPDXDocumentView( BaseDetailView, ): def get(self, request, *args, **kwargs): - instance = self.get_object() - spdx_document = self.get_spdx_document(instance, self.request.user) - document_name = spdx_document.as_dict()["name"] - filename = f"{document_name}.spdx.json" + spdx_document = outputs.get_spdx_document(self.get_object(), self.request.user) + spdx_document_json = spdx_document.as_json() - if not spdx_document: - raise Http404 - - response = FileResponse( - spdx_document.as_json(), - filename=filename, + return outputs.get_attachment_response( + file_content=spdx_document_json, + filename=outputs.get_spdx_filename(spdx_document), content_type="application/json", ) - response["Content-Disposition"] = f'attachment; filename="{filename}"' - - return response - - @staticmethod - def get_spdx_document(instance, user): - spdx_packages = instance.get_spdx_packages() - - creation_info = spdx.CreationInfo( - person_name=f"{user.first_name} {user.last_name}", - person_email=user.email, - organization_name=user.dataspace.name, - tool=f"DejaCode-{dejacode_version}", - ) - - document = spdx.Document( - name=f"dejacode_{instance.dataspace.name}_{instance._meta.model_name}_{instance}", - namespace=f"https://dejacode.com/spdxdocs/{instance.uuid}", - creation_info=creation_info, - packages=[package.as_spdx() for package in spdx_packages], - extracted_licenses=get_spdx_extracted_licenses(spdx_packages), - ) - - return document class ExportCycloneDXBOMView( @@ -2396,57 +2337,11 @@ class ExportCycloneDXBOMView( ): def get(self, request, *args, **kwargs): instance = self.get_object() - cyclonedx_bom = self.get_cyclonedx_bom(instance, self.request.user) - base_filename = f"dejacode_{instance.dataspace.name}_{instance._meta.model_name}" - filename = safe_filename(f"{base_filename}_{instance}.cdx.json") - - if not cyclonedx_bom: - raise Http404 - - outputter = cyclonedx_output.get_instance( - bom=cyclonedx_bom, - output_format=cyclonedx_output.OutputFormat.JSON, - ) - bom_json = outputter.output_as_string() + cyclonedx_bom = outputs.get_cyclonedx_bom(instance, self.request.user) + cyclonedx_bom_json = outputs.get_cyclonedx_bom_json(cyclonedx_bom) - response = FileResponse( - bom_json, - filename=filename, + return outputs.get_attachment_response( + file_content=cyclonedx_bom_json, + filename=outputs.get_cyclonedx_filename(instance), content_type="application/json", ) - response["Content-Disposition"] = f'attachment; filename="{filename}"' - - return response - - @staticmethod - def get_cyclonedx_bom(instance, user): - """https://cyclonedx.org/use-cases/#dependency-graph""" - cyclonedx_components = [] - - if hasattr(instance, "get_cyclonedx_components"): - cyclonedx_components = [ - component.as_cyclonedx() for component in instance.get_cyclonedx_components() - ] - - bom = cyclonedx_bom.Bom(components=cyclonedx_components) - - cdx_component = instance.as_cyclonedx() - cdx_component.dependencies.update([component.bom_ref for component in cyclonedx_components]) - - bom.metadata = cyclonedx_bom.BomMetaData( - component=cdx_component, - tools=[ - cyclonedx_bom.Tool( - vendor="nexB", - name="DejaCode", - version=dejacode_version, - ) - ], - authors=[ - cyclonedx_bom.OrganizationalContact( - name=f"{user.first_name} {user.last_name}", - ) - ], - ) - - return bom diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 0000000..7450989 --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,17 @@ +.. _faq: + +FAQs +==== + +You can't find what you're looking for? Below you'll find answers to a few of +our frequently asked questions. + +.. _faq_download_sbom: + +How can I download SBOM for my products? +---------------------------------------- + +**CycloneDX SBOM and SPDX document** can be downloaded from the "Share" menu of +the product details view in the web UI or from dedicated endpoint URLs of the REST API. + +Refer to :ref:`how_to_3` for more details. diff --git a/docs/howto-3.rst b/docs/howto-3.rst new file mode 100644 index 0000000..0129d8c --- /dev/null +++ b/docs/howto-3.rst @@ -0,0 +1,26 @@ +.. _how_to_3: + +============================================= +How To 3 - Downloading SBOM for Your Products +============================================= + +You can obtain both **CycloneDX and SPDX Software Bill of Materials (SBOM)** documents +either through the web user interface (UI) or via the REST API endpoints. + +Web User Interface +================== + +1. Navigate to the product details view. +2. Click on the "Share" menu. +3. Download the desired SBOM format from the available options. + +REST API Endpoints +================== + +You can programmatically fetch the SBOMs using the following dedicated endpoint URLs of +the REST API: + +- CycloneDX: ``/api/v2/products/{uuid}/cyclonedx_sbom/`` +- SPDX: ``/api/v2/products/{uuid}/spdx_document/`` + +Replace ``{uuid}`` with the unique identifier of your product. diff --git a/docs/index.rst b/docs/index.rst index 7f9fccf..7615ca9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ Welcome to the very start of your DejaCode journey! installation dataspace application-settings + faq .. toctree:: :maxdepth: 1 @@ -26,6 +27,7 @@ Welcome to the very start of your DejaCode journey! howto-1 howto-2 + howto-3 .. toctree:: :maxdepth: 1 diff --git a/product_portfolio/api.py b/product_portfolio/api.py index 45a2230..206e78f 100644 --- a/product_portfolio/api.py +++ b/product_portfolio/api.py @@ -20,6 +20,7 @@ from component_catalog.api import PackageEmbeddedSerializer from component_catalog.api import ValidateLicenseExpressionMixin from component_catalog.license_expression_dje import clean_related_expression +from dje import outputs from dje.api import CreateRetrieveUpdateListViewSet from dje.api import DataspacedAPIFilterSet from dje.api import DataspacedHyperlinkedRelatedField @@ -33,6 +34,7 @@ from dje.filters import MultipleUUIDFilter from dje.filters import NameVersionFilter from dje.permissions import assign_all_object_permissions +from dje.views import SendAboutFilesMixin from product_portfolio.filters import ComponentCompletenessAPIFilter from product_portfolio.forms import ImportFromScanForm from product_portfolio.forms import ImportManifestsForm @@ -268,7 +270,7 @@ class PullProjectDataSerializer(serializers.Serializer): ) -class ProductViewSet(CreateRetrieveUpdateListViewSet): +class ProductViewSet(SendAboutFilesMixin, CreateRetrieveUpdateListViewSet): queryset = Product.objects.none() serializer_class = ProductSerializer filterset_class = ProductFilterSet @@ -400,6 +402,36 @@ def pull_scancodeio_project_data(self, request, *args, **kwargs): return Response({"status": "Packages import from ScanCode.io in progress..."}) + @action(detail=True, name="Download AboutCode files") + def aboutcode_files(self, request, uuid): + instance = self.get_object() + about_files = instance.get_about_files() + filename = self.get_filename(instance) + return self.get_zipped_response(about_files, filename) + + @action(detail=True, name="Download SPDX document") + def spdx_document(self, request, uuid): + spdx_document = outputs.get_spdx_document(self.get_object(), self.request.user) + spdx_document_json = spdx_document.as_json() + + return outputs.get_attachment_response( + file_content=spdx_document_json, + filename=outputs.get_spdx_filename(spdx_document), + content_type="application/json", + ) + + @action(detail=True, name="Download CycloneDX SBOM") + def cyclonedx_sbom(self, request, uuid): + instance = self.get_object() + cyclonedx_bom = outputs.get_cyclonedx_bom(instance, self.request.user) + cyclonedx_bom_json = outputs.get_cyclonedx_bom_json(cyclonedx_bom) + + return outputs.get_attachment_response( + file_content=cyclonedx_bom_json, + filename=outputs.get_cyclonedx_filename(instance), + content_type="application/json", + ) + class BaseProductRelationSerializer(ValidateLicenseExpressionMixin, DataspacedSerializer): product = NameVersionHyperlinkedRelatedField( diff --git a/product_portfolio/tests/test_api.py b/product_portfolio/tests/test_api.py index 9c6bd1c..877e1ab 100644 --- a/product_portfolio/tests/test_api.py +++ b/product_portfolio/tests/test_api.py @@ -486,6 +486,57 @@ def test_api_product_endpoint_pull_scancodeio_project_data_action(self, mock_get self.assertEqual(expected, response.data) self.assertEqual(1, ScanCodeProject.objects.count()) + def test_api_product_endpoint_aboutcode_files_action(self): + url = reverse("api_v2:product-aboutcode-files", args=[self.product1.uuid]) + + self.client.login(username=self.base_user.username, password="secret") + response = self.client.get(url) + self.assertEqual(status.HTTP_404_NOT_FOUND, response.status_code) + + # Required permissions + add_perm(self.base_user, "add_product") + assign_perm("view_product", self.base_user, self.product1) + + response = self.client.get(url) + self.assertEqual(status.HTTP_200_OK, response.status_code) + expected = 'attachment; filename="p1_about.zip"' + self.assertEqual(expected, response["Content-Disposition"]) + self.assertEqual("application/zip", response["Content-Type"]) + + def test_api_product_endpoint_spdx_document_action(self): + url = reverse("api_v2:product-spdx-document", args=[self.product1.uuid]) + + self.client.login(username=self.base_user.username, password="secret") + response = self.client.get(url) + self.assertEqual(status.HTTP_404_NOT_FOUND, response.status_code) + + # Required permissions + add_perm(self.base_user, "add_product") + assign_perm("view_product", self.base_user, self.product1) + + response = self.client.get(url) + self.assertEqual(status.HTTP_200_OK, response.status_code) + expected = 'attachment; filename="dejacode_nexb_product_p1.spdx.json"' + self.assertEqual(expected, response["Content-Disposition"]) + self.assertEqual("application/json", response["Content-Type"]) + + def test_api_product_endpoint_cyclonedx_sbom_action(self): + url = reverse("api_v2:product-cyclonedx-sbom", args=[self.product1.uuid]) + + self.client.login(username=self.base_user.username, password="secret") + response = self.client.get(url) + self.assertEqual(status.HTTP_404_NOT_FOUND, response.status_code) + + # Required permissions + add_perm(self.base_user, "add_product") + assign_perm("view_product", self.base_user, self.product1) + + response = self.client.get(url) + self.assertEqual(status.HTTP_200_OK, response.status_code) + expected = 'attachment; filename="dejacode_nexb_product_p1.cdx.json"' + self.assertEqual(expected, response["Content-Disposition"]) + self.assertEqual("application/json", response["Content-Type"]) + class ProductRelatedAPITestCase(TestCase): def setUp(self): diff --git a/product_portfolio/tests/test_views.py b/product_portfolio/tests/test_views.py index 823bbf9..afe7764 100644 --- a/product_portfolio/tests/test_views.py +++ b/product_portfolio/tests/test_views.py @@ -28,18 +28,16 @@ from component_catalog.models import ComponentAssignedPackage from component_catalog.models import ComponentKeyword from component_catalog.models import Package -from dejacode import __version__ as dejacode_version from dejacode_toolkit import scancodeio from dje.models import Dataspace from dje.models import History +from dje.outputs import get_spdx_extracted_licenses from dje.tasks import logger as tasks_logger from dje.tasks import pull_project_data_from_scancodeio from dje.tasks import scancodeio_submit_project from dje.tests import add_perms from dje.tests import create_superuser from dje.tests import create_user -from dje.views import ExportSPDXDocumentView -from dje.views import get_spdx_extracted_licenses from license_library.models import License from organization.models import Owner from policy.models import UsagePolicy @@ -2460,28 +2458,6 @@ def test_product_portfolio_product_export_spdx_view(self): ) self.assertEqual("application/json", response.headers["Content-Type"]) - document = ExportSPDXDocumentView.get_spdx_document(self.product1, self.super_user) - document.creation_info.created = "2000-01-01T01:02:03Z" - expected = { - "spdxVersion": "SPDX-2.3", - "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", - "name": "dejacode_nexb_product_product1_with_space_1.0", - "documentNamespace": f"https://dejacode.com/spdxdocs/{self.product1.uuid}", - "creationInfo": { - "created": "2000-01-01T01:02:03Z", - "creators": [ - "Person: (user@email.com)", - "Organization: nexB ()", - f"Tool: DejaCode-{dejacode_version}", - ], - "licenseListVersion": "3.18", - }, - "packages": [], - "documentDescribes": [], - } - self.assertEqual(expected, document.as_dict()) - def test_product_portfolio_product_export_spdx_get_spdx_extracted_licenses(self): owner1 = Owner.objects.create(name="Owner1", dataspace=self.dataspace) license1 = License.objects.create(