From 971856c36f2ddb3701546ed40286a2a0ca7eae89 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 2 May 2023 22:08:30 +0530 Subject: [PATCH 1/3] Add support for npm priority queue Signed-off-by: Keshav Priyadarshi --- minecode/visitors/npm.py | 73 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/minecode/visitors/npm.py b/minecode/visitors/npm.py index afcc342b..a77e9983 100644 --- a/minecode/visitors/npm.py +++ b/minecode/visitors/npm.py @@ -10,14 +10,17 @@ import logging import json +import requests from packageurl import PackageURL from packagedcode.npm import npm_api_url from packagedcode.npm import split_scoped_package_name +from packagedcode.npm import NpmPackageJsonHandler from minecode import seed from minecode import visit_router +from minecode import priority_router from minecode.visitors import NonPersistentHttpVisitor from minecode.visitors import URI @@ -103,3 +106,73 @@ def get_uris(self, content): data=json.dumps(doc, separators=(',', ':'), ensure_ascii=False), # note: visited is True since there nothing more to visit visited=True) + +def get_package_json(namespace, name, version): + """ + Return the contents of the package.json file of the package described by the purl + field arguments in a string. + """ + # Create URLs using purl fields + url = npm_api_url( + namespace=namespace, + name=name, + version=version, + ) + + try: + response = requests.get(url) + response.raise_for_status() + return response.json() + except requests.exceptions.HTTPError as err: + print(f"HTTP error occurred: {err}") + + +def map_npm_package(package_url): + """ + Add a npm `package_url` to the PackageDB. + + Return an error string if any errors are encountered during the process + """ + from minecode.management.commands.priority_queue import add_package_to_scan_queue + from minecode.management.commands.run_map import merge_or_create_package + + pacakge_json = get_package_json( + namespace = package_url.namespace, + name=package_url.name, + version=package_url.version, + ) + + if not pacakge_json: + error = f'Package does not exist on npmjs: {package_url}' + logger.error(error) + return error + + package = NpmPackageJsonHandler._parse( + json_data=pacakge_json + ) + + db_package, _, _, _ = merge_or_create_package(package, visit_level=0) + # Submit package for scanning + if db_package: + add_package_to_scan_queue(db_package) + + +@priority_router.route('pkg:npm/.*') +def process_request(purl_str): + """ + Process `priority_resource_uri` containing a npm Package URL (PURL) as a + URI. + + This involves obtaining Package information for the PURL from npm and + using it to create a new PackageDB entry. The package is then added to the + scan queue afterwards. + """ + package_url = PackageURL.from_string(purl_str) + if not package_url.version: + return + + error_msg = map_npm_package(package_url) + + if error_msg: + return error_msg + From eeb05effc7b9ce5262d39605056798aec1025af0 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 3 May 2023 00:26:42 +0530 Subject: [PATCH 2/3] Add test for npm priority queue Signed-off-by: Keshav Priyadarshi --- minecode/tests/test_npm.py | 46 ++++++++++- .../npm/lodash_package-expected.json | 82 +++++++++++++++++++ 2 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 minecode/tests/testfiles/npm/lodash_package-expected.json diff --git a/minecode/tests/test_npm.py b/minecode/tests/test_npm.py index 34c56b12..745aabe2 100644 --- a/minecode/tests/test_npm.py +++ b/minecode/tests/test_npm.py @@ -13,15 +13,18 @@ import os import re +from django.test import TestCase as DjangoTestCase from mock import Mock from mock import patch +from packagedcode.npm import NpmPackageJsonHandler +from packageurl import PackageURL -from minecode.utils_test import mocked_requests_get -from minecode.utils_test import JsonBasedTesting - +import packagedb from minecode import mappers from minecode import route from minecode.models import ResourceURI +from minecode.utils_test import JsonBasedTesting +from minecode.utils_test import mocked_requests_get from minecode.visitors import npm @@ -166,3 +169,40 @@ def test_regex_npm_mapper(self): regex = re.compile(r'^https://registry.npmjs.org/[^\/]+$') result = re.match(regex, 'https://registry.npmjs.org/react-mobile-navigation-modal') self.assertTrue(result) + +class NpmPriorityQueueTests(JsonBasedTesting, DjangoTestCase): + test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') + + def setUp(self): + super(NpmPriorityQueueTests, self).setUp() + expected_json_loc = self.get_test_loc('npm/lodash_package-expected.json') + with open(expected_json_loc) as f: + self.expected_json_contents = json.load(f) + + self.scan_package = NpmPackageJsonHandler._parse( + json_data=self.expected_json_contents, + ) + + def test_get_package_json(self, regen=False): + json_contents = npm.get_package_json( + namespace=self.scan_package.namespace, + name=self.scan_package.name, + version=self.scan_package.version + ) + if regen: + with open(self.expected_pom_loc, 'w') as f: + f.write(json_contents) + self.assertEqual(self.expected_json_contents, json_contents) + + def test_map_npm_package(self): + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(0, package_count) + package_url = PackageURL.from_string(self.scan_package.purl) + npm.map_npm_package(package_url) + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(1, package_count) + package = packagedb.models.Package.objects.all().first() + expected_purl_str = 'pkg:npm/lodash@4.17.21' + expected_download_url = 'https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz' + self.assertEqual(expected_purl_str, package.purl) + self.assertEqual(expected_download_url, package.download_url) diff --git a/minecode/tests/testfiles/npm/lodash_package-expected.json b/minecode/tests/testfiles/npm/lodash_package-expected.json new file mode 100644 index 00000000..32a34466 --- /dev/null +++ b/minecode/tests/testfiles/npm/lodash_package-expected.json @@ -0,0 +1,82 @@ +{ + "name":"lodash", + "version":"4.17.21", + "description":"Lodash modular utilities.", + "keywords":[ + "modules", + "stdlib", + "util" + ], + "homepage":"https://lodash.com/", + "repository":{ + "type":"git", + "url":"git+https://github.com/lodash/lodash.git" + }, + "icon":"https://lodash.com/icon.svg", + "license":"MIT", + "main":"lodash.js", + "author":{ + "name":"John-David Dalton", + "email":"john.david.dalton@gmail.com" + }, + "contributors":[ + { + "name":"John-David Dalton", + "email":"john.david.dalton@gmail.com" + }, + { + "name":"Mathias Bynens", + "email":"mathias@qiwi.be" + } + ], + "scripts":{ + "test":"echo \"See https://travis-ci.org/lodash-archive/lodash-cli for testing details.\"" + }, + "gitHead":"c6e281b878b315c7a10d90f9c2af4cdb112d9625", + "bugs":{ + "url":"https://github.com/lodash/lodash/issues" + }, + "_id":"lodash@4.17.21", + "_nodeVersion":"14.15.5", + "_npmVersion":"6.14.11", + "dist":{ + "integrity":"sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "shasum":"679591c564c3bffaae8454cf0b3df370c3d6911c", + "tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "fileCount":1054, + "unpackedSize":1412415, + "npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgMS3ZCRA9TVsSAnZWagAA8+4P/jx+SJ6Ue5oAJjz0L7gw\nLDD5YvP8aoliFq4GYkwUXfVQvOwomIPfa+U5Kao/hDfuwFQ/Bq5D5nSsl2bj\nrjJgvlKXna0SId8AgDgY2fB7zSfninuJvalY4iTWMN8DFSpG0XE2QFfoKpd3\njDmuzcNtgr79QV6DgjOVkHiP1IGNDlLTc1QEKiwo/5CdGQi1q/iCj6dViQMJ\nByuuuV2Qzi3f/FI25cG797WZar1MHhhlcnB50HiVBGp54IZOyuqdqWPduZQo\nvhONtonxPGBm3/J+uAkeUSSyL3Ud+FzLvdg8WEI9gDL0yvU4k0FcsnOONEYn\nngLaKEsw2xAnPBYW3Lf73Jnpwx6FAT3k49kgzxiNYSxEo7x4wiuNtBoDMyNw\nEKj6SZ0bUNmaJgiMfDnnDjCKjI3JrO1hho8z6CkwuvxuWLlW9wSsVayggzAI\nEhfeTeISugVHh332oDY2MI/Ysu8MnVN8fGmqeYQBBFj3aWatuA2NvVjACnX/\n54G7FtCU8TxZpm9shFRSopBx8PeI3r+icx1CT8YVFypY416PLnidHyqtME1G\neuRd1nWEz18hvVUAEHmuvHo+EPP3tITmTTUPQcZGMdBcZC+4UBmPMWX466HE\nbHw4aOnUWMa0sWfsERC5xzRZAb4lgMPEoTOnZyN4usMy7x9TzGZKZvU24HUE\nmpae\r\n=NOmG\r\n-----END PGP SIGNATURE-----\r\n", + "signatures":[ + { + "keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA", + "sig":"MEUCIF3Yithbtmy1aEBNlfNWbLswAfPIyQUuNUGARD3Ex2t4AiEA6TlN2ZKJCUpS/Sf2Z6MduF1BNSvayHIpu5wAcICcKXw=" + } + ] + }, + "_npmUser":{ + "name":"bnjmnt4n", + "email":"benjamin@dev.ofcr.se" + }, + "directories":{ + + }, + "maintainers":[ + { + "name":"mathias", + "email":"mathias@qiwi.be" + }, + { + "name":"jdalton", + "email":"john.david.dalton@gmail.com" + }, + { + "name":"bnjmnt4n", + "email":"benjamin@dev.ofcr.se" + } + ], + "_npmOperationalInternal":{ + "host":"s3://npm-registry-packages", + "tmp":"tmp/lodash_4.17.21_1613835736675_0.01913912595366596" + }, + "_hasShrinkwrap":false +} \ No newline at end of file From 4ce38fb525de74891f27b4df21deb2ad27c20e73 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 3 May 2023 15:19:05 +0530 Subject: [PATCH 3/3] Fix typo Signed-off-by: Keshav Priyadarshi --- minecode/visitors/npm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/minecode/visitors/npm.py b/minecode/visitors/npm.py index a77e9983..405032db 100644 --- a/minecode/visitors/npm.py +++ b/minecode/visitors/npm.py @@ -124,7 +124,7 @@ def get_package_json(namespace, name, version): response.raise_for_status() return response.json() except requests.exceptions.HTTPError as err: - print(f"HTTP error occurred: {err}") + logger.error(f"HTTP error occurred: {err}") def map_npm_package(package_url): @@ -136,19 +136,19 @@ def map_npm_package(package_url): from minecode.management.commands.priority_queue import add_package_to_scan_queue from minecode.management.commands.run_map import merge_or_create_package - pacakge_json = get_package_json( + package_json = get_package_json( namespace = package_url.namespace, name=package_url.name, version=package_url.version, ) - if not pacakge_json: + if not package_json: error = f'Package does not exist on npmjs: {package_url}' logger.error(error) return error package = NpmPackageJsonHandler._parse( - json_data=pacakge_json + json_data=package_json ) db_package, _, _, _ = merge_or_create_package(package, visit_level=0)