diff --git a/Makefile b/Makefile index 63f0cc28..e1d5693b 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,8 @@ check: check_docs clean: @echo "-> Clean the Python env" @PYTHON_EXECUTABLE=${PYTHON_EXE} ./configure --clean + rm -rf .venv/ .*cache/ *.egg-info/ build/ dist/ + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete migrate: @echo "-> Apply database migrations" diff --git a/docs/source/how-to-guides/symbols_and_strings.rst b/docs/source/how-to-guides/symbols_and_strings.rst index ca4bf8e8..be401564 100644 --- a/docs/source/how-to-guides/symbols_and_strings.rst +++ b/docs/source/how-to-guides/symbols_and_strings.rst @@ -9,8 +9,7 @@ from codebase resources. .. note:: This tutorial assumes that you have a working installation of PurlDB. - - If you don't, please refer to the :ref:`installation`page. + If you don't, please refer to the :ref:`installation` page. diff --git a/docs/source/matchcode/matchcode-pipeline.rst b/docs/source/matchcode/matchcode-pipeline.rst index 9dc70233..49ffd706 100644 --- a/docs/source/matchcode/matchcode-pipeline.rst +++ b/docs/source/matchcode/matchcode-pipeline.rst @@ -8,9 +8,8 @@ code matching on an archive of files. .. note:: This tutorial assumes that you have a working installation of PurlDB. If you + don't, please refer to the :ref:`installation` page. - don't, please refer to the :ref:`installation`page. - Throughout this tutorial, we will use ``pkg:npm/deep-equal@1.0.1`` and a modified copy of ``index.js`` from it. diff --git a/packagedb/api.py b/packagedb/api.py index 01ccfabe..c453d010 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -17,6 +17,7 @@ from django.forms.fields import MultipleChoiceField import django_filters +from aboutcode.federatedcode.contrib.django import utils from django_filters.filters import Filter from django_filters.filters import MultipleChoiceFilter from django_filters.filters import OrderingFilter @@ -34,6 +35,7 @@ from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.throttling import AnonRateThrottle +from rest_framework.views import APIView from univers.version_constraint import InvalidConstraintsError from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange @@ -48,6 +50,7 @@ from minecode.route import NoRouteAvailable from packagedb.filters import PackageSearchFilter from packagedb.models import Package +from packagedb.models import PackageActivity from packagedb.models import PackageContentType from packagedb.models import PackageSet from packagedb.models import PackageWatch @@ -59,6 +62,7 @@ from packagedb.serializers import DependentPackageSerializer from packagedb.serializers import IndexPackagesResponseSerializer from packagedb.serializers import IndexPackagesSerializer +from packagedb.serializers import PackageActivitySerializer from packagedb.serializers import PackageAPISerializer from packagedb.serializers import PackageSetAPISerializer from packagedb.serializers import PackageWatchAPISerializer @@ -1409,3 +1413,30 @@ def get_all_versions(purl): pkg_type: range_class.version_class for pkg_type, range_class in RANGE_CLASS_BY_SCHEMES.items() } + + +class PackageActivityViewSet(viewsets.ReadOnlyModelViewSet): + queryset = PackageActivity.objects.get_queryset().order_by("-creation_date") + serializer_class = PackageActivitySerializer + lookup_field = "uuid" + + +class PackageActivityListenerView(APIView): + def post(self, request): + activity_type = utils.get_package_activity_type(request.data) + content = utils.get_package_activity_content(request.data) + author = utils.get_package_activity_author(request.data) + update_date = utils.get_package_activity_update_date(request.data) + + if content and activity_type.lower() == "create": + PackageActivity.objects.create( + author=author, + content=content, + activity_update_date=update_date, + ) + return Response(status=status.HTTP_200_OK) + + return Response( + {"error": "Invalid JSON"}, + status=status.HTTP_400_BAD_REQUEST, + ) diff --git a/packagedb/management/commands/subscribe_package.py b/packagedb/management/commands/subscribe_package.py new file mode 100644 index 00000000..aadf864e --- /dev/null +++ b/packagedb/management/commands/subscribe_package.py @@ -0,0 +1,44 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# PurlDB is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +from django.core.management.base import BaseCommand + +from aboutcode.federatedcode.client import subscribe_package + +from purldb_project import settings + + +class Command(BaseCommand): + help = "Subscribe package for their metadata update from FederatedCode." + + def add_arguments(self, parser): + parser.add_argument( + "purl", type=str, help="Specify a PURL to subscribe for updates." + ) + + def handle(self, *args, **options): + purl = options.get("purl") + + federatedcode_host = settings.FEDERATEDCODE_HOST_URL + remote_username = settings.FEDERATEDCODE_PURLDB_REMOTE_USERNAME + + if not (federatedcode_host and remote_username): + raise ValueError("FederatedCode env variable not configured.") + + response = subscribe_package(federatedcode_host, remote_username, purl) + + style = self.style.ERROR + if response.status_code == 200: + style = self.style.SUCCESS + + self.stdout.write( + response.text, + style, + ) diff --git a/packagedb/migrations/0089_packageactivity.py b/packagedb/migrations/0089_packageactivity.py new file mode 100644 index 00000000..16e50c81 --- /dev/null +++ b/packagedb/migrations/0089_packageactivity.py @@ -0,0 +1,70 @@ +# Generated by Django 5.1.2 on 2024-12-11 12:52 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("packagedb", "0088_alter_package_md5_alter_package_sha1_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="PackageActivity", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + help_text="The identifier of the Package set", + unique=True, + ), + ), + ( + "author", + models.CharField( + help_text="Author of package activity.", max_length=300 + ), + ), + ("content", models.JSONField(help_text="Package activity content.")), + ( + "activity_update_date", + models.DateTimeField( + blank=True, + db_index=True, + help_text="Timestamp indicating when original activity was last updated.", + null=True, + ), + ), + ( + "creation_date", + models.DateTimeField( + auto_now_add=True, + help_text="Timestamp indicating when this object was created.", + ), + ), + ( + "is_processed", + models.BooleanField( + default=False, + help_text="True if this activity has been processed.", + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/packagedb/models.py b/packagedb/models.py index 60493a29..452526e0 100644 --- a/packagedb/models.py +++ b/packagedb/models.py @@ -27,6 +27,9 @@ from django.utils.translation import gettext_lazy as _ import natsort +from aboutcode.federatedcode.contrib.django.models import ( + FederatedCodePackageActivityMixin, +) from dateutil.parser import parse as dateutil_parse from licensedcode.cache import build_spdx_license_expression from packagedcode.models import normalize_qualifiers @@ -1466,3 +1469,22 @@ def create_auth_token(sender, instance=None, created=False, **kwargs): """Create an API key token on user creation, using the signal system.""" if created: Token.objects.get_or_create(user_id=instance.pk) + + +class PackageActivity(FederatedCodePackageActivityMixin): + """Record of package activity from a FederatedCode.""" + + uuid = models.UUIDField( + default=uuid.uuid4, + unique=True, + editable=False, + help_text=_("The identifier of the package activity"), + ) + creation_date = models.DateTimeField( + auto_now_add=True, + help_text=_("Timestamp indicating when this object was created."), + ) + + is_processed = models.BooleanField( + default=False, help_text=_("True if this activity has been processed.") + ) diff --git a/packagedb/serializers.py b/packagedb/serializers.py index fbdb2973..983fa897 100644 --- a/packagedb/serializers.py +++ b/packagedb/serializers.py @@ -25,6 +25,7 @@ from packagedb.models import DependentPackage from packagedb.models import Package +from packagedb.models import PackageActivity from packagedb.models import PackageSet from packagedb.models import PackageWatch from packagedb.models import Party @@ -554,3 +555,20 @@ def is_supported_sort_field(field): # A field could have a leading `-` return field.lstrip("-") in PACKAGE_FILTER_SORT_FIELDS + + +class PackageActivitySerializer(ModelSerializer): + url = HyperlinkedIdentityField( + view_name="api:packageactivity-detail", lookup_field="uuid" + ) + + class Meta: + model = PackageActivity + fields = [ + "url", + "author", + "content", + "activity_update_date", + "creation_date", + "is_processed", + ] diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index d7c150b8..ce054919 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -25,6 +25,7 @@ from minecode.tests import FIXTURES_REGEN from minecode.utils_test import JsonBasedTesting from packagedb.models import Package +from packagedb.models import PackageActivity from packagedb.models import PackageContentType from packagedb.models import PackageSet from packagedb.models import PackageWatch @@ -1705,3 +1706,65 @@ def test_to_golang_purl(self): ) expected = "pkg:golang/github.com/gorilla/mux@v1.7.3" self.assertEqual(expected, response.data["package_url"]) + + +class PackageActivityAPITestCase(JsonBasedTesting, TestCase): + def setUp(self): + self.data = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://www.aboutcode.org/ns/federatedcode", + ], + "type": "Create", + "actor": { + "id": "https://127.0.0.1:8000/api/v0/purls/@pkg:npm/atlasboard/", + "type": "Package", + "name": "root", + "purl": "pkg:npm/atlasboard", + "inbox": "https://127.0.0.1:8000/api/v0/purls/@pkg:npm/atlasboard/inbox", + "outbox": "https://127.0.0.1:8000/api/v0/purls/@pkg:npm/atlasboard/outbox", + "followers": "https://127.0.0.1:8000/api/v0/purls/@pkg:npm/atlasboard/followers/", + "publicKey": { + "id": "https://127.0.0.1:8000/api/v0/purls/@pkg:npm/atlasboard/", + "owner": "https://127.0.0.1:8000/api/v0/users/@root", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----...-----END PUBLIC KEY-----", + }, + }, + "object": { + "id": "https://127.0.0.1:8000/notes/f9d10718-c6a2-4414-a96c-6cfcafe17be9", + "type": "Note", + "author": "pkg:npm/atlasboard@127.0.0.1:8000", + "content": "purl: pkg:npm/atlasboard@1.1.11\nscans:\n - tool: pkg:pypi/scancode-toolkit\n file_name: scancodeio.json\n", + "update_date": "2024-12-19 10:49:26.201915+00:00", + }, + "to": [], + "cc": "https://www.w3.org/ns/activitystreams#Public", + } + self.client = APIClient() + + def test_api_package_activity_listener_inbox_endpoint(self): + response = self.client.post( + "/api/users/@purldb/inbox", data=self.data, format="json" + ) + self.assertEqual(status.HTTP_200_OK, response.status_code) + + def test_api_package_activity_creation(self): + package_activity_count = PackageActivity.objects.count() + self.assertEqual(0, package_activity_count) + + response = self.client.post( + "/api/users/@purldb/inbox", data=self.data, format="json" + ) + self.assertEqual(status.HTTP_200_OK, response.status_code) + + package_activity_count = PackageActivity.objects.count() + self.assertEqual(1, package_activity_count) + + def test_api_package_activity_endpoint(self): + response = self.client.post( + "/api/users/@purldb/inbox", data=self.data, format="json" + ) + self.assertEqual(status.HTTP_200_OK, response.status_code) + + package_activity = self.client.get("/api/package_activity/") + self.assertEqual(1, package_activity.data.get("count")) diff --git a/purldb_project/settings.py b/purldb_project/settings.py index aacea674..39fbd4b4 100644 --- a/purldb_project/settings.py +++ b/purldb_project/settings.py @@ -322,3 +322,10 @@ if not PURLDB_ASYNC: for queue_config in RQ_QUEUES.values(): queue_config["ASYNC"] = False + +# FederatedCode integration + +FEDERATEDCODE_HOST_URL = env.str("FEDERATEDCODE_HOST_URL", default="") +FEDERATEDCODE_PURLDB_REMOTE_USERNAME = env.str( + "FEDERATEDCODE_PURLDB_REMOTE_USERNAME", default="purldb" +) diff --git a/purldb_project/urls.py b/purldb_project/urls.py index 2e590a00..f27f1d35 100644 --- a/purldb_project/urls.py +++ b/purldb_project/urls.py @@ -22,6 +22,8 @@ from minecode.api import ScannableURIViewSet from minecode.api import index_package_scan from packagedb.api import CollectViewSet +from packagedb.api import PackageActivityListenerView +from packagedb.api import PackageActivityViewSet from packagedb.api import PackageSetViewSet from packagedb.api import PackageUpdateSet from packagedb.api import PackageViewSet @@ -46,6 +48,7 @@ api_router.register( "approximate_directory_structure_index", ApproximateDirectoryStructureIndexViewSet ) +api_router.register("package_activity", PackageActivityViewSet) urlpatterns = [ @@ -70,5 +73,15 @@ ), ] + +# Endpoint to receive updates related to subscribed packages +urlpatterns.append( + path( + "api/users/@purldb/inbox", + PackageActivityListenerView.as_view(), + name="package_activity_listener", + ), +) + if settings.DEBUG and settings.DEBUG_TOOLBAR: urlpatterns.append(path("__debug__/", include("debug_toolbar.urls"))) diff --git a/requirements.txt b/requirements.txt index e45d9fbc..1450a7e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -aboutcode-toolkit==11.0.0 -aboutcode.hashid==0.1.0 +aboutcode.federatedcode == 0.1.0 aboutcode.pipeline==0.2.0 +aboutcode-toolkit==11.0.0 +aboutcode.hashid==0.2.0 arrow==1.3.0 asgiref==3.8.1 attrs==24.2.0 @@ -23,12 +24,12 @@ construct==2.10.70 container-inspector==33.0.0 crispy-bootstrap3==2024.1 crontab==1.0.1 -cryptography==44.0.0 +cryptography==43.0.0 cyclonedx-python-lib==8.5.0 debian_inspector==31.1.0 defusedxml==0.7.1 -Deprecated==1.2.15 -Django==5.1.3 +Deprecated==1.2.14 +Django==5.1.4 django-crispy-forms==2.3 django-environ==0.11.2 django-filter==24.3 @@ -76,7 +77,7 @@ license-expression==30.4.0 lxml==5.3.0 Markdown==3.6 markdown-it-py==3.0.0 -MarkupSafe==3.0.2 +MarkupSafe==2.1.5 matchcode-toolkit==7.0.0 mdurl==0.1.2 mock==5.1.0 @@ -86,7 +87,7 @@ normality==2.5.0 openpyxl==3.1.5 packagedcode-msitools==0.101.210706 packageurl-python==0.16.0 -packaging==24.2 +packaging==24.1 packvers==21.5 parameter-expansion-patched==0.3.1 pdfminer.six==20240706 @@ -117,8 +118,8 @@ python-dotenv==1.0.1 python-inspector==0.12.1 pytz==2024.2 PyYAML==6.0.2 -rdflib==7.1.1 -redis==5.2.0 +rdflib==7.0.0 +redis==5.2.1 referencing==0.35.1 regipy==5.0.0 reppy2==0.3.6 @@ -132,11 +133,11 @@ rubymarshal==1.0.3 samecode==0.5.1 saneyaml==0.6.1 scancode-toolkit==32.3.0 -scancodeio @ git+https://github.com/aboutcode-org/scancode.io.git@3eeb0c040c6077d773680b59da6fa283ba1500bf +# scancodeio==34.9.2 semantic-version==2.10.0 semver==3.0.2 setuptools==75.6.0 -six==1.17.0 +six==1.16.0 smmap==5.0.1 sortedcontainers==2.4.0 soupsieve==2.6 diff --git a/setup.cfg b/setup.cfg index 9837eb2d..a7718494 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = bitarray == 2.9.2 debian-inspector == 31.1.0 commoncode == 32.1.0 - Django == 5.1.3 + Django == 5.1.4 django-environ == 0.11.2 django-rq == 3.0.0 djangorestframework == 3.15.2 @@ -63,9 +63,12 @@ install_requires = matchcode-toolkit == 7.0.0 purl2vcs == 2.0.0 univers == 30.12.1 - scancodeio @ git+https://github.com/aboutcode-org/scancode.io.git@3eeb0c040c6077d773680b59da6fa283ba1500bf + scancodeio @ git+https://github.com/aboutcode-org/scancode.io@e1607c7f9684dbca9cf67bbbbf3145e0aaf06fdf + # scancodeio == 34.9.2 gitpython == 3.1.43 samecode == 0.5.1 + # FederatedCode integration + aboutcode.federatedcode == 0.1.0 setup_requires = setuptools_scm[toml] >= 4 python_requires = >=3.8