diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c035cef..5b77ab4f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,10 +1,10 @@ Changelog ========= -next-version +v4.0.0 ------------ -- TBD +- Add `/api/docs` Swagger API documentation for API endpoints. v3.0.0 ------- diff --git a/packagedb/api.py b/packagedb/api.py index 25002569..d11be44d 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -17,12 +17,26 @@ from django_filters.filters import Filter from django_filters.filters import OrderingFilter from django_filters.rest_framework import FilterSet +from drf_spectacular.utils import OpenApiParameter +from drf_spectacular.utils import extend_schema +from packageurl import PackageURL +from packageurl.contrib.django.utils import purl_to_lookups +from rest_framework import mixins +from rest_framework import status +from rest_framework import viewsets +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.throttling import AnonRateThrottle +from univers.version_constraint import InvalidConstraintsError +from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.version_range import VersionRange +from univers.versions import InvalidVersion + from matchcode.api import MultipleCharFilter from matchcode.api import MultipleCharInFilter - -from minecode import priority_router # UnusedImport here! # But importing the mappers and visitors module triggers routes registration +from minecode import priority_router from minecode import visitors # NOQA from minecode.models import PriorityResourceURI from minecode.models import ScannableURI @@ -37,6 +51,8 @@ from packagedb.package_managers import get_api_package_name from packagedb.package_managers import get_version_fetcher from packagedb.serializers import DependentPackageSerializer +from packagedb.serializers import IndexPackagesResponseSerializer +from packagedb.serializers import IndexPackagesSerializer from packagedb.serializers import PackageAPISerializer from packagedb.serializers import PackageSetAPISerializer from packagedb.serializers import PackageWatchAPISerializer @@ -47,23 +63,11 @@ from packagedb.serializers import PurlValidateSerializer from packagedb.serializers import ResourceAPISerializer from packagedb.throttling import StaffUserRateThrottle -from packageurl import PackageURL -from packageurl.contrib.django.utils import purl_to_lookups -from rest_framework import mixins -from rest_framework import status -from rest_framework import viewsets -from rest_framework.decorators import action -from rest_framework.response import Response -from rest_framework.throttling import AnonRateThrottle -from univers.version_constraint import InvalidConstraintsError -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import VersionRange -from univers.versions import InvalidVersion logger = logging.getLogger(__name__) -class CreateListRetrieveUpdateViewSet( +class CreateListRetrieveUpdateViewSetMixin( mixins.CreateModelMixin, mixins.ListModelMixin, mixins.RetrieveModelMixin, @@ -550,7 +554,12 @@ class PackageSetViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = PackageSetAPISerializer -class PackageWatchViewSet(CreateListRetrieveUpdateViewSet): +class PackageWatchViewSet(CreateListRetrieveUpdateViewSetMixin): + """ + Take a `purl` and periodically watch for the new version of the package. + Add the new package version to the scan queue. + Default watch interval is 7 days. + """ queryset = PackageWatch.objects.get_queryset().order_by('-id') serializer_class = PackageWatchAPISerializer lookup_field = 'package_url' @@ -570,11 +579,14 @@ class CollectViewSet(viewsets.ViewSet): If the package does not exist, we will fetch the Package data and return it in the same request. - - **Note:** Use `Index packages` for bulk indexing of packages; use `Reindex packages` - for bulk reindexing of existing packages. + + **Note:** Use `Index packages` for bulk indexing/reindexing of packages. """ - + serializer_class=None + @extend_schema( + parameters=[OpenApiParameter('purl', str, 'query', description='PackageURL')], + responses={200:PackageAPISerializer()}, + ) def list(self, request, format=None): purl = request.query_params.get('purl') @@ -604,14 +616,20 @@ def list(self, request, format=None): message = {} if errors: message = { - 'status': f'error(s) occured when fetching metadata for {purl}: {errors}' + 'status': f'error(s) occurred when fetching metadata for {purl}: {errors}' } return Response(message) serializer = PackageAPISerializer(packages, many=True, context={'request': request}) return Response(serializer.data) - - @action(detail=False, methods=['post']) + + @extend_schema( + request=IndexPackagesSerializer, + responses={ + 200: IndexPackagesResponseSerializer(), + }, + ) + @action(detail=False, methods=['post'], serializer_class=IndexPackagesSerializer) def index_packages(self, request, *args, **kwargs): """ Take a list of `packages` (where each item is a dictionary containing either PURL @@ -624,7 +642,7 @@ def index_packages(self, request, *args, **kwargs): **Note:** When a versionless PURL is supplied without a vers range, then all the versions of that package will be considered for indexing/reindexing. - **Input example:** + **Request example:** { "packages": [ @@ -676,10 +694,16 @@ def _reindex_package(package, reindexed_packages): return package.rescan() reindexed_packages.append(package) + + serializer = self.serializer_class(data=request.data) - packages = request.data.get('packages') or [] - reindex = request.data.get('reindex') or False - reindex_set = request.data.get('reindex_set') or False + if not serializer.is_valid(): + return Response({'errors': serializer.errors}, status=400) + + validated_data = serializer.validated_data + packages = validated_data.get('packages', []) + reindex = validated_data.get('reindex', False) + reindex_set = validated_data.get('reindex_set', False) queued_packages = [] unqueued_packages = [] @@ -733,7 +757,9 @@ def _reindex_package(package, reindexed_packages): 'unsupported_vers_count': len(unsupported_vers), 'unsupported_vers': unsupported_vers, } - return Response(response_data) + + serializer = IndexPackagesResponseSerializer(response_data, context={'request': request}) + return Response(serializer.data) class PurlValidateViewSet(viewsets.ViewSet): @@ -745,7 +771,7 @@ class PurlValidateViewSet(viewsets.ViewSet): `gem`, `golang`, `hex`, `maven`, `npm`, `nuget` and `pypi` ecosystems. **Example request:** - ```doc + ``` GET /api/validate/?purl=pkg:npm/foobar@12.3.1&check_existence=false ``` @@ -761,6 +787,13 @@ class PurlValidateViewSet(viewsets.ViewSet): def get_view_name(self): return 'Validate PURL' + @extend_schema( + parameters=[ + OpenApiParameter('purl', str, 'query', description='PackageURL'), + OpenApiParameter('check_existence', bool, 'query', description='Check existence', default=False), + ], + responses={200: PurlValidateResponseSerializer()}, + ) def list(self, request): serializer = self.serializer_class(data=request.query_params) diff --git a/packagedb/serializers.py b/packagedb/serializers.py index fceb9eb3..06b89567 100644 --- a/packagedb/serializers.py +++ b/packagedb/serializers.py @@ -20,7 +20,9 @@ from rest_framework.serializers import HyperlinkedIdentityField from rest_framework.serializers import HyperlinkedModelSerializer from rest_framework.serializers import HyperlinkedRelatedField +from rest_framework.serializers import IntegerField from rest_framework.serializers import JSONField +from rest_framework.serializers import ListField from rest_framework.serializers import ModelSerializer from rest_framework.serializers import Serializer from rest_framework.serializers import SerializerMethodField @@ -366,6 +368,43 @@ class Meta: fields = ['depth', 'watch_interval', 'is_active'] +class PackageVersSerializer(Serializer): + purl = CharField() + vers = CharField(required=False) + + +class IndexPackagesSerializer(Serializer): + packages = PackageVersSerializer(many=True) + reindex = BooleanField(default=False) + reindex_set = BooleanField(default=False) + + +class IndexPackagesResponseSerializer(Serializer): + queued_packages_count = IntegerField(help_text="Number of package urls placed on the index queue.") + queued_packages = ListField( + child=CharField(), + help_text="List of package urls that were placed on the index queue." + ) + requeued_packages_count = IntegerField(help_text="Number of existing package urls placed on the rescan queue.") + requeued_packages = ListField( + child=CharField(), + help_text="List of existing package urls that were placed on the rescan queue." + ) + unqueued_packages_count = IntegerField(help_text="Number of package urls not placed on the index queue.") + unqueued_packages = ListField( + child=CharField(), + help_text="List of package urls that were not placed on the index queue." + ) + unsupported_packages_count = IntegerField(help_text="Number of package urls that are not processable by the index queue.") + unsupported_packages = ListField( + child=CharField(), + help_text="List of package urls that are not processable by the index queue." + ) + unsupported_vers_count = IntegerField(help_text="Number of vers range that are not supported by the univers or package_manager.") + unsupported_vers = ListField( + child=CharField(), + help_text="List of vers range that are not supported by the univers or package_manager." + ) class PurlValidateResponseSerializer(Serializer): valid = BooleanField() exists = BooleanField(required=False) @@ -374,4 +413,4 @@ class PurlValidateResponseSerializer(Serializer): class PurlValidateSerializer(Serializer): purl = CharField(required=True) - check_existence = BooleanField(required=False, default=False) \ No newline at end of file + check_existence = BooleanField(required=False, default=False) diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 6507e2c6..94cba66f 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -1096,7 +1096,7 @@ def test_api_resource_checksum_filter(self): self.resource1.name, self.resource2.name, ]) - self.assertEquals(expected_names, names) + self.assertEqual(expected_names, names) filters = f'?sha1={self.resource1.sha1}&sha1={self.resource2.sha1}' response = self.client.get(f'/api/resources/{filters}') @@ -1106,7 +1106,7 @@ def test_api_resource_checksum_filter(self): self.resource1.name, self.resource2.name, ]) - self.assertEquals(expected_names, names) + self.assertEqual(expected_names, names) class PurlValidateApiTestCase(TestCase): @@ -1140,15 +1140,15 @@ def test_api_purl_validation(self): } response2 = self.client.get(f"/api/validate/", data=data2) - self.assertEquals(True, response1.data["valid"]) - self.assertEquals(True, response1.data["exists"]) - self.assertEquals( + self.assertEqual(True, response1.data["valid"]) + self.assertEqual(True, response1.data["exists"]) + self.assertEqual( "The provided Package URL is valid, and the package exists in the upstream repo.", response1.data["message"], ) - self.assertEquals(False, response2.data["valid"]) - self.assertEquals( + self.assertEqual(False, response2.data["valid"]) + self.assertEqual( "The provided PackageURL is not valid.", response2.data["message"] ) @@ -1160,11 +1160,11 @@ def test_api_purl_validation_unsupported_package_type(self): response1 = self.client.get(f"/api/validate/", data=data1) - self.assertEquals(True, response1.data["valid"]) - self.assertEquals( + self.assertEqual(True, response1.data["valid"]) + self.assertEqual( "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", response1.data["message"] ) - self.assertEquals(None, response1.data["exists"]) + self.assertEqual(None, response1.data["exists"]) def test_api_purl_validation_empty_request(self): data1 = {} diff --git a/purldb_project/__init__.py b/purldb_project/__init__.py index 0eef5774..e0b80baa 100644 --- a/purldb_project/__init__.py +++ b/purldb_project/__init__.py @@ -10,6 +10,8 @@ import os import sys +__version__ = "3.0.0" + def command_line(): '''Command line entry point.''' diff --git a/purldb_project/settings.py b/purldb_project/settings.py index 0dbbb2ec..6678e411 100644 --- a/purldb_project/settings.py +++ b/purldb_project/settings.py @@ -11,7 +11,9 @@ from pathlib import Path import environ +from purldb_project import __version__ +PURLDB_VERSION = __version__ PROJECT_DIR = Path(__file__).resolve().parent ROOT_DIR = PROJECT_DIR.parent @@ -73,6 +75,7 @@ # Third-party apps 'django_filters', 'rest_framework', + 'drf_spectacular', 'rest_framework.authtoken', 'django_rq', ) @@ -258,6 +261,7 @@ 'DEFAULT_THROTTLE_RATES': REST_FRAMEWORK_DEFAULT_THROTTLE_RATES, 'EXCEPTION_HANDLER': 'packagedb.throttling.throttled_exception_handler', 'DEFAULT_PAGINATION_CLASS': 'packagedb.api_custom.PageSizePagination', + 'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', # Limit the load on the Database returning a small number of records by default. https://github.com/nexB/vulnerablecode/issues/819 "PAGE_SIZE": 20, } @@ -293,11 +297,17 @@ "127.0.0.1", ] -# Active seeders: each active seeder class need to be added explictly here +# Active seeders: each active seeder class need to be added explicitly here ACTIVE_SEEDERS = [ 'minecode.visitors.maven.MavenSeed', ] +SPECTACULAR_SETTINGS = { + 'TITLE': 'PurlDB API', + 'DESCRIPTION': 'Tools to create and expose a database of purls (Package URLs)', + 'VERSION': PURLDB_VERSION, + 'SERVE_INCLUDE_SCHEMA': False, +} RQ_QUEUES = { 'default': { "HOST": env.str("PURLDB_REDIS_HOST", default="localhost"), diff --git a/purldb_project/urls.py b/purldb_project/urls.py index 902d7c38..63a867ed 100644 --- a/purldb_project/urls.py +++ b/purldb_project/urls.py @@ -26,6 +26,8 @@ from minecode.api import PriorityResourceURIViewSet from packagedb.api import PurlValidateViewSet from packagedb.api import CollectViewSet +from drf_spectacular.views import SpectacularAPIView +from drf_spectacular.views import SpectacularSwaggerView api_router = routers.DefaultRouter() @@ -49,5 +51,7 @@ TemplateView.as_view(template_name='robots.txt', content_type='text/plain'), ), path('api/', include((api_router.urls, 'api'))), - path('', RedirectView.as_view(url='api/')), + path("", RedirectView.as_view(url="api/")), + path('api/schema/', SpectacularAPIView.as_view(), name='schema'), + path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'), ] diff --git a/setup.cfg b/setup.cfg index 0fec309c..30566b83 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,6 +44,7 @@ install_requires = django-rq == 2.10.1 djangorestframework == 3.14.0 django-filter == 23.5 + drf-spectacular == 0.26.5 fetchcode == 0.3.0 gunicorn == 21.2.0 ftputil == 5.0.4