From 15f53b00442603675174127b4dbf624b1f69ab2a Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 28 Jul 2023 17:31:08 -0700 Subject: [PATCH 1/8] Create endpoint for package checksum filtering Signed-off-by: Jono Yang --- packagedb/api.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 330eb723..ab5ac8aa 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -218,7 +218,7 @@ def resources(self, request, *args, **kwargs): paginated_qs = self.paginate_queryset(qs) serializer = ResourceAPISerializer(paginated_qs, many=True, context={'request': request}) - return Response(serializer.data) + return self.get_paginated_response(serializer.data) @action(detail=False) def get_package(self, request, *args, **kwargs): @@ -241,7 +241,7 @@ def get_package(self, request, *args, **kwargs): return Response({}) serializer = PackageAPISerializer(packages, many=True, context={'request': request}) - return Response(serializer.data) + return self.get_paginated_response(serializer.data) @action(detail=False) def get_or_fetch_package(self, request, *args, **kwargs): @@ -284,7 +284,7 @@ def get_or_fetch_package(self, request, *args, **kwargs): return Response(message) serializer = PackageAPISerializer(packages, many=True, context={'request': request}) - return Response(serializer.data) + return self.get_paginated_response(serializer.data) @action(detail=True) def get_enhanced_package_data(self, request, *args, **kwargs): @@ -409,6 +409,30 @@ def _reindex_package(package, reindexed_packages): } return Response(response_data) + @action(detail=False, methods=['post']) + def filter_by_checksums(self, request, *args, **kwargs): + unsupported_fields = [] + for field, value in request.data.items(): + if field not in ('md5', 'sha1', 'sha256', 'sha512'): + unsupported_fields.append(field) + + if unsupported_fields: + unsupported_fields_str = ', '.join(unsupported_fields) + response_data = { + 'status': f'Unsupported field(s) given: {unsupported_fields_str}' + } + return Response(response_data) + + q = Q() + for field, value in request.data.items(): + d = {f'{field}__in': value} + q |= Q(**d) + + qs = Package.objects.filter(q) + paginated_qs = self.paginate_queryset(qs) + serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request}) + return self.get_paginated_response(serializer.data) + UPDATEABLE_FIELDS = [ 'primary_language', From 95b768c4a3b1073d6b21f72d28b4d9795ee0c077 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 31 Jul 2023 18:21:31 -0700 Subject: [PATCH 2/8] Add filter_by_checksums to resource API * Create tests Signed-off-by: Jono Yang --- minecode/utils_test.py | 7 +- packagedb/api.py | 64 +++++++- packagedb/tests/test_api.py | 36 ++++- .../package-filter_by_checksums-expected.json | 149 ++++++++++++++++++ ...resource-filter_by_checksums-expected.json | 70 ++++++++ 5 files changed, 315 insertions(+), 11 deletions(-) create mode 100644 packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json create mode 100644 packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json diff --git a/minecode/utils_test.py b/minecode/utils_test.py index 86731b4c..9da25fd1 100644 --- a/minecode/utils_test.py +++ b/minecode/utils_test.py @@ -25,6 +25,7 @@ from django.db.migrations.executor import MigrationExecutor from django.test import TestCase as DjangoTestCase from rest_framework.utils.serializer_helpers import ReturnDict +from rest_framework.utils.serializer_helpers import ReturnList from commoncode.testcase import FileBasedTesting from scancode.cli_test_utils import purl_with_fake_uuid @@ -206,13 +207,13 @@ def _normalize_results(self, data, fields_to_remove=[]): with `purl_with_fake_uuid()` and fields from `fields_to_remove` have been removed from `data`. """ - if type(data) == list: + if type(data) in (list, ReturnList): return [self._normalize_results(entry, fields_to_remove) for entry in data] if type(data) in (dict, OrderedDict, ReturnDict): normalized_data = {} for key, value in data.items(): - if type(value) in [list, dict, OrderedDict, ReturnDict]: + if type(value) in (list, ReturnList, dict, OrderedDict, ReturnDict): value = self._normalize_results(value, fields_to_remove) if ( key in ("package_uid", "dependency_uid", "for_package_uid") @@ -229,7 +230,7 @@ def _normalize_results(self, data, fields_to_remove=[]): return data def _remove_fields_from_results(self, data, fields_to_remove): - if type(data) == list: + if type(data) in (list, ReturnList): return [self._remove_fields_from_results(entry, fields_to_remove) for entry in data] if type(data) in (dict, OrderedDict, ReturnDict): diff --git a/packagedb/api.py b/packagedb/api.py index ab5ac8aa..456d48d9 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -87,6 +87,46 @@ class ResourceViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = ResourceFilter lookup_field = 'sha1' + @action(detail=False, methods=['post']) + def filter_by_checksums(self, request, *args, **kwargs): + """ + Take a mapping, where the keys are the names of the checksum algorthm + and the values is a list of checksum values and query those values + against the packagedb. + + Example: + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ] + } + """ + data = dict(request.data) + unsupported_fields = [] + for field, value in data.items(): + if field not in ('md5', 'sha1'): + unsupported_fields.append(field) + + if unsupported_fields: + unsupported_fields_str = ', '.join(unsupported_fields) + response_data = { + 'status': f'Unsupported field(s) given: {unsupported_fields_str}' + } + return Response(response_data) + + q = Q() + for field, value in data.items(): + # We create this intermediate dictionary so we can modify the field + # name to have __in at the end + d = {f'{field}__in': value} + q |= Q(**d) + + qs = Resource.objects.filter(q) + paginated_qs = self.paginate_queryset(qs) + serializer = ResourceAPISerializer(paginated_qs, many=True, context={'request': request}) + return self.get_paginated_response(serializer.data) + class MultiplePackageURLFilter(Filter): def filter(self, qs, value): @@ -241,7 +281,7 @@ def get_package(self, request, *args, **kwargs): return Response({}) serializer = PackageAPISerializer(packages, many=True, context={'request': request}) - return self.get_paginated_response(serializer.data) + return Response(serializer.data) @action(detail=False) def get_or_fetch_package(self, request, *args, **kwargs): @@ -284,7 +324,7 @@ def get_or_fetch_package(self, request, *args, **kwargs): return Response(message) serializer = PackageAPISerializer(packages, many=True, context={'request': request}) - return self.get_paginated_response(serializer.data) + return Response(serializer.data) @action(detail=True) def get_enhanced_package_data(self, request, *args, **kwargs): @@ -411,8 +451,22 @@ def _reindex_package(package, reindexed_packages): @action(detail=False, methods=['post']) def filter_by_checksums(self, request, *args, **kwargs): + """ + Take a mapping, where the keys are the names of the checksum algorthm + and the values is a list of checksum values and query those values + against the packagedb. + + Example: + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ] + } + """ + data = dict(request.data) unsupported_fields = [] - for field, value in request.data.items(): + for field, value in data.items(): if field not in ('md5', 'sha1', 'sha256', 'sha512'): unsupported_fields.append(field) @@ -424,7 +478,9 @@ def filter_by_checksums(self, request, *args, **kwargs): return Response(response_data) q = Q() - for field, value in request.data.items(): + for field, value in data.items(): + # We create this intermediate dictionary so we can modify the field + # name to have __in at the end d = {f'{field}__in': value} q |= Q(**d) diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index ec1a3e43..0df2c252 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -27,7 +27,8 @@ from packagedb.models import Resource -class ResourceAPITestCase(TestCase): +class ResourceAPITestCase(JsonBasedTesting, TestCase): + test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') def setUp(self): self.package1 = Package.objects.create( @@ -194,8 +195,21 @@ def test_api_resource_list_endpoint_filters_by_package2_purl(self): self.assertEqual(test_resource.get('extra_data'), self.resource2.extra_data) self.assertEqual(test_resource.get('type'), self.resource2.type) + def test_api_resource_filter_by_checksums(self): + sha1s = [ + 'testsha11', + 'testsha12', + ] + data = { + 'sha1': sha1s + } + response = self.client.post('/api/resources/filter_by_checksums/', data=data) + self.assertEqual(2, response.data['count']) + expected = self.get_test_loc('api/resource-filter_by_checksums-expected.json') + self.check_expected_results(response.data['results'], expected, regen=True) -class PackageApiTestCase(TestCase): +class PackageApiTestCase(JsonBasedTesting, TestCase): + test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') def setUp(self): @@ -348,9 +362,9 @@ def test_api_package_resources_action(self): response = self.client.get(reverse('api:package-resources', args=[self.package.uuid])) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(10, len(response.data)) + self.assertEqual(10, response.data['count']) - for result, i in zip(response.data, range(0, 10)): + for result, i in zip(response.data['results'], range(0, 10)): self.assertEqual(result.get('path'), 'path{}/'.format(i)) def test_api_package_list_endpoint_multiple_char_filters(self): @@ -429,6 +443,20 @@ def test_package_api_index_packages_endpoint(self): ] self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages']) + def test_package_api_filter_by_checksums(self): + sha1s = [ + 'testsha1', + 'testsha1-2', + 'testsha1-3', + ] + data = { + 'sha1': sha1s + } + response = self.client.post('/api/packages/filter_by_checksums/', data=data) + self.assertEqual(3, response.data['count']) + expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') + self.check_expected_results(response.data['results'], expected, regen=True) + class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json new file mode 100644 index 00000000..d614cf2d --- /dev/null +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json @@ -0,0 +1,149 @@ +[ + { + "url":"http://testserver/api/packages/c47b9874-a870-4bfb-9578-e65e79cc4ac9/", + "uuid":"c47b9874-a870-4bfb-9578-e65e79cc4ac9", + "filename":"Foo.zip", + "package_sets":[], + "package_content":null, + "purl":"pkg:generic/generic/foo@12.34?test_qual=qual#test_subpath", + "type":"generic", + "namespace":"generic", + "name":"foo", + "version":"12.34", + "qualifiers":"test_qual=qual", + "subpath":"test_subpath", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://example.com", + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "size":101, + "md5":"testmd5", + "sha1":"testsha1", + "sha256":null, + "sha512":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "package_uid":"pkg:generic/generic/foo@12.34?test_qual=qual&uuid=fixed-uid-done-for-testing-5642512d1758#test_subpath", + "datasource_id":null, + "file_references":[], + "dependencies":[], + "resources":"http://testserver/api/packages/c47b9874-a870-4bfb-9578-e65e79cc4ac9/resources/" + }, + { + "url":"http://testserver/api/packages/63897eec-1475-471c-b46d-aa7066eed476/", + "uuid":"63897eec-1475-471c-b46d-aa7066eed476", + "filename":"Bar.zip", + "package_sets":[], + "package_content":null, + "purl":"pkg:npm/example/bar@56.78", + "type":"npm", + "namespace":"example", + "name":"bar", + "version":"56.78", + "qualifiers":"", + "subpath":"", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://somethingelse.org", + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "size":100, + "md5":"testmd5-2", + "sha1":"testsha1-2", + "sha256":null, + "sha512":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "package_uid":"pkg:npm/example/bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "file_references":[], + "dependencies":[], + "resources":"http://testserver/api/packages/63897eec-1475-471c-b46d-aa7066eed476/resources/" + }, + { + "url":"http://testserver/api/packages/f327655d-bdd5-4fb7-bf06-e0b4bc596120/", + "uuid":"f327655d-bdd5-4fb7-bf06-e0b4bc596120", + "filename":"Baz.zip", + "package_sets":[], + "package_content":null, + "purl":"pkg:jar/sample/baz@90.12", + "type":"jar", + "namespace":"sample", + "name":"baz", + "version":"90.12", + "qualifiers":"", + "subpath":"", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://anotherexample.com", + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "size":100, + "md5":"testmd5-3", + "sha1":"testsha1-3", + "sha256":null, + "sha512":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "package_uid":"pkg:jar/sample/baz@90.12?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "file_references":[], + "dependencies":[], + "resources":"http://testserver/api/packages/f327655d-bdd5-4fb7-bf06-e0b4bc596120/resources/" + } +] \ No newline at end of file diff --git a/packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json new file mode 100644 index 00000000..a5f83a44 --- /dev/null +++ b/packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json @@ -0,0 +1,70 @@ +[ + { + "package":"http://testserver/api/packages/7c2697c2-30a0-4838-b2f2-07c5ad9f3d72/", + "purl":"pkg:type1/name1", + "path":"package1/contents1.txt", + "type":"file", + "name":"", + "extension":"", + "size":101, + "md5":"testmd51", + "sha1":"testsha11", + "sha256":"testsha2561", + "sha512":"testsha5121", + "git_sha1":"testgit_sha11", + "mime_type":"", + "file_type":"", + "programming_language":"", + "is_binary":false, + "is_text":false, + "is_archive":false, + "is_media":false, + "is_key_file":false, + "detected_license_expression":"", + "detected_license_expression_spdx":"", + "license_detections":[], + "license_clues":[], + "percentage_of_license_text":null, + "copyrights":[], + "holders":[], + "authors":[], + "package_data":[], + "emails":[], + "urls":[], + "extra_data":"{\"test1\": \"data1\"}" + }, + { + "package":"http://testserver/api/packages/505f6476-108c-41e2-850f-9720e33c07cb/", + "purl":"pkg:type2/name2", + "path":"package2/contents2.txt", + "type":"file", + "name":"", + "extension":"", + "size":102, + "md5":"testmd52", + "sha1":"testsha12", + "sha256":"testsha2562", + "sha512":"testsha5122", + "git_sha1":"testgit_sha12", + "mime_type":"", + "file_type":"", + "programming_language":"", + "is_binary":false, + "is_text":false, + "is_archive":false, + "is_media":false, + "is_key_file":false, + "detected_license_expression":"", + "detected_license_expression_spdx":"", + "license_detections":[], + "license_clues":[], + "percentage_of_license_text":null, + "copyrights":[], + "holders":[], + "authors":[], + "package_data":[], + "emails":[], + "urls":[], + "extra_data":"{\"test2\": \"data2\"}" + } +] \ No newline at end of file From 3d3330b259ee4840244c6d5d7c4ee23b3bdf06dd Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 1 Aug 2023 11:59:41 -0700 Subject: [PATCH 3/8] Increase page size to 100 Signed-off-by: Jono Yang --- packagedb/api_custom.py | 2 +- purldb/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packagedb/api_custom.py b/packagedb/api_custom.py index 5686311f..05d73da2 100644 --- a/packagedb/api_custom.py +++ b/packagedb/api_custom.py @@ -17,6 +17,6 @@ class PageSizePagination(PageNumberPagination): For example: http://api.example.org/accounts/?page=4&page_size=100 """ - page_size = 10 + page_size = 100 max_page_size = 100 page_size_query_param = 'page_size' diff --git a/purldb/settings.py b/purldb/settings.py index 648ed3da..5852266a 100644 --- a/purldb/settings.py +++ b/purldb/settings.py @@ -238,7 +238,7 @@ ), 'DEFAULT_PAGINATION_CLASS': 'packagedb.api_custom.PageSizePagination', # Limit the load on the Database returning a small number of records by default. https://github.com/nexB/vulnerablecode/issues/819 - "PAGE_SIZE": 10, + "PAGE_SIZE": 100, } if not PURLDB_REQUIRE_AUTHENTICATION: From cfbbc3bfec2e8bd926b393bd7308a519aa4575aa Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 2 Aug 2023 08:47:21 -0700 Subject: [PATCH 4/8] Set SECURE_PROXY_SSL_HEADER Signed-off-by: Jono Yang --- purldb/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/purldb/settings.py b/purldb/settings.py index 5852266a..e12b5dde 100644 --- a/purldb/settings.py +++ b/purldb/settings.py @@ -89,6 +89,10 @@ WSGI_APPLICATION = "purldb.wsgi.application" +SECURE_PROXY_SSL_HEADER = env.tuple( + "SECURE_PROXY_SSL_HEADER", default=("HTTP_X_FORWARDED_PROTO", "https") +) + # Database DATABASES = { From 1a4929af9c3e07e659e8fb1c69bc031327350a6e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 2 Aug 2023 19:00:32 -0700 Subject: [PATCH 5/8] Update scan_and_fingerprint_package * Use new fingerprinting functions Signed-off-by: Jono Yang --- .../pipelines/scan_and_fingerprint_package.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/matchcode-toolkit/src/matchcode_toolkit/pipelines/scan_and_fingerprint_package.py b/matchcode-toolkit/src/matchcode_toolkit/pipelines/scan_and_fingerprint_package.py index 7bc5e13b..0f15d13a 100644 --- a/matchcode-toolkit/src/matchcode_toolkit/pipelines/scan_and_fingerprint_package.py +++ b/matchcode-toolkit/src/matchcode_toolkit/pipelines/scan_and_fingerprint_package.py @@ -20,10 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints - from scanpipe.pipelines.scan_package import ScanPackage -from scanpipe.pipes.codebase import ProjectCodebase +from scanpipe.pipes import matchcode class ScanAndFingerprintPackage(ScanPackage): @@ -62,5 +60,4 @@ def fingerprint_codebase(self): """ Compute directory fingerprints for matching purposes """ - project_codebase = ProjectCodebase(self.project) - compute_codebase_directory_fingerprints(project_codebase) + matchcode.fingerprint_codebase_directories(self.project) From ecfefe7e7d94ec9ac0fd4ecbfb8370dfcd71a1f2 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 2 Aug 2023 19:03:38 -0700 Subject: [PATCH 6/8] Update scan_and_fingerprint_package pipeline Signed-off-by: Jono Yang --- matchcode-toolkit/CHANGELOG.rst | 5 +++++ matchcode-toolkit/setup.cfg | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/matchcode-toolkit/CHANGELOG.rst b/matchcode-toolkit/CHANGELOG.rst index ee235eea..541ddfa8 100644 --- a/matchcode-toolkit/CHANGELOG.rst +++ b/matchcode-toolkit/CHANGELOG.rst @@ -1,6 +1,11 @@ Changelog ========= +v1.1.2 +------ + +*2023-08-02* -- Update ``scan_and_fingerprint_package`` pipeline to use new directory fingerprinting functions from scancode.io. + v1.1.1 ------ diff --git a/matchcode-toolkit/setup.cfg b/matchcode-toolkit/setup.cfg index 9d0f6c84..4cfec167 100644 --- a/matchcode-toolkit/setup.cfg +++ b/matchcode-toolkit/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = matchcode-toolkit -version = 1.1.1 +version = 1.1.2 license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 From 5e19a89abd99d3b5bd87cd11059ba7a125b3848d Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 3 Aug 2023 14:25:54 -0700 Subject: [PATCH 7/8] Update docstrings Signed-off-by: Jono Yang --- packagedb/api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packagedb/api.py b/packagedb/api.py index 456d48d9..194b4d98 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -94,6 +94,10 @@ def filter_by_checksums(self, request, *args, **kwargs): and the values is a list of checksum values and query those values against the packagedb. + Supported checksum fields are: + - md5 + - sha1 + Example: { "sha1": [ @@ -456,6 +460,12 @@ def filter_by_checksums(self, request, *args, **kwargs): and the values is a list of checksum values and query those values against the packagedb. + Supported checksum fields are: + - md5 + - sha1 + - sha256 + - sha512 + Example: { "sha1": [ From 13f6b7f0856b5ca20f4e0c36ac9125f2457fbf84 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 3 Aug 2023 15:05:42 -0700 Subject: [PATCH 8/8] Fix test * Exclude correct fields, do not regen tests Signed-off-by: Jono Yang --- packagedb/api.py | 26 +++++++++++++++++++ packagedb/tests/test_api.py | 4 +-- .../package-filter_by_checksums-expected.json | 15 +++-------- ...resource-filter_by_checksums-expected.json | 2 -- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 194b4d98..43df7aa3 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -105,6 +105,19 @@ def filter_by_checksums(self, request, *args, **kwargs): "27afff2610b5a94274a2311f8b15e514446b0e76 ] } + + Multiple checksums algorithms can be passed together: + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ], + "md5": [ + "e927df60b093456d4e611ae235c1aa5b" + ] + } + + This will return Resources whose sha1 or md5 matches those values. """ data = dict(request.data) unsupported_fields = [] @@ -473,6 +486,19 @@ def filter_by_checksums(self, request, *args, **kwargs): "27afff2610b5a94274a2311f8b15e514446b0e76 ] } + + Multiple checksums algorithms can be passed together: + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ], + "md5": [ + "e927df60b093456d4e611ae235c1aa5b" + ] + } + + This will return Packages whose sha1 or md5 matches those values. """ data = dict(request.data) unsupported_fields = [] diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 0df2c252..8311e9e4 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -206,7 +206,7 @@ def test_api_resource_filter_by_checksums(self): response = self.client.post('/api/resources/filter_by_checksums/', data=data) self.assertEqual(2, response.data['count']) expected = self.get_test_loc('api/resource-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, regen=True) + self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "package"], regen=False) class PackageApiTestCase(JsonBasedTesting, TestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') @@ -455,7 +455,7 @@ def test_package_api_filter_by_checksums(self): response = self.client.post('/api/packages/filter_by_checksums/', data=data) self.assertEqual(3, response.data['count']) expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, regen=True) + self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=False) class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json index d614cf2d..a01c743b 100644 --- a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json @@ -1,7 +1,5 @@ [ { - "url":"http://testserver/api/packages/c47b9874-a870-4bfb-9578-e65e79cc4ac9/", - "uuid":"c47b9874-a870-4bfb-9578-e65e79cc4ac9", "filename":"Foo.zip", "package_sets":[], "package_content":null, @@ -45,12 +43,9 @@ "package_uid":"pkg:generic/generic/foo@12.34?test_qual=qual&uuid=fixed-uid-done-for-testing-5642512d1758#test_subpath", "datasource_id":null, "file_references":[], - "dependencies":[], - "resources":"http://testserver/api/packages/c47b9874-a870-4bfb-9578-e65e79cc4ac9/resources/" + "dependencies":[] }, { - "url":"http://testserver/api/packages/63897eec-1475-471c-b46d-aa7066eed476/", - "uuid":"63897eec-1475-471c-b46d-aa7066eed476", "filename":"Bar.zip", "package_sets":[], "package_content":null, @@ -94,12 +89,9 @@ "package_uid":"pkg:npm/example/bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", "datasource_id":null, "file_references":[], - "dependencies":[], - "resources":"http://testserver/api/packages/63897eec-1475-471c-b46d-aa7066eed476/resources/" + "dependencies":[] }, { - "url":"http://testserver/api/packages/f327655d-bdd5-4fb7-bf06-e0b4bc596120/", - "uuid":"f327655d-bdd5-4fb7-bf06-e0b4bc596120", "filename":"Baz.zip", "package_sets":[], "package_content":null, @@ -143,7 +135,6 @@ "package_uid":"pkg:jar/sample/baz@90.12?uuid=fixed-uid-done-for-testing-5642512d1758", "datasource_id":null, "file_references":[], - "dependencies":[], - "resources":"http://testserver/api/packages/f327655d-bdd5-4fb7-bf06-e0b4bc596120/resources/" + "dependencies":[] } ] \ No newline at end of file diff --git a/packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json index a5f83a44..325beb1c 100644 --- a/packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json +++ b/packagedb/tests/testfiles/api/resource-filter_by_checksums-expected.json @@ -1,6 +1,5 @@ [ { - "package":"http://testserver/api/packages/7c2697c2-30a0-4838-b2f2-07c5ad9f3d72/", "purl":"pkg:type1/name1", "path":"package1/contents1.txt", "type":"file", @@ -34,7 +33,6 @@ "extra_data":"{\"test1\": \"data1\"}" }, { - "package":"http://testserver/api/packages/505f6476-108c-41e2-850f-9720e33c07cb/", "purl":"pkg:type2/name2", "path":"package2/contents2.txt", "type":"file",