From fed9e9bdcb8ea1cc3f8eb8fdd49ccd2556e7ee27 Mon Sep 17 00:00:00 2001 From: yma Date: Fri, 1 Apr 2022 07:37:21 +0800 Subject: [PATCH 1/5] Fix dist missing issue for npm package installing --- charon/constants.py | 2 ++ charon/utils/archive.py | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/charon/constants.py b/charon/constants.py index 03c38c19..7b3e99a0 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -173,3 +173,5 @@ PROD_INFO_SUFFIX = ".prodinfo" MANIFEST_SUFFIX = ".txt" DEFAULT_ERRORS_LOG = "errors.log" + +NRRC_REGISTRY = "npm.registry.redhat.com" diff --git a/charon/utils/archive.py b/charon/utils/archive.py index ae6f6f3c..7c485a19 100644 --- a/charon/utils/archive.py +++ b/charon/utils/archive.py @@ -21,9 +21,11 @@ import tempfile import shutil from enum import Enum -from json import load, JSONDecodeError +from json import load, JSONDecodeError, dump from typing import Tuple from zipfile import ZipFile, is_zipfile +from charon.constants import NRRC_REGISTRY +from charon.utils.files import digest, HashType logger = logging.getLogger(__name__) @@ -54,7 +56,7 @@ def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool) -> Tupl tgz.extractall() for f in tgz: if f.name.endswith("package.json"): - parse_paths = __parse_npm_package_version_paths(f.path) + version_data, parse_paths = __parse_npm_package_version_paths(f.path) package_name_path = parse_paths[0] os.makedirs(os.path.join(target_dir, parse_paths[0])) tarball_parent_path = os.path.join(target_dir, parse_paths[0], "-") @@ -63,7 +65,11 @@ def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool) -> Tupl target_dir, parse_paths[0], parse_paths[1] ) valid_paths.append(os.path.join(version_metadata_parent_path, "package.json")) + if is_for_upload: + tgz_relative_path = "/".join([parse_paths[0], "-", _get_tgz_name(path)]) + __write_npm_version_dist(path, f.path, version_data, tgz_relative_path) + os.makedirs(tarball_parent_path) target = os.path.join(tarball_parent_path, os.path.basename(path)) shutil.copyfile(path, target) @@ -81,12 +87,33 @@ def _get_tgz_name(path: str): return "" -def __parse_npm_package_version_paths(path: str) -> list: +def _del_none(d): + for key, value in list(d.items()): + if value is None: + del d[key] + elif isinstance(value, dict): + _del_none(value) + return d + + +def __write_npm_version_dist(path: str, version_meta_extract_path: str, version_data: dict, + tgz_relative_path: str): + tarball_link = "".join(["https://", NRRC_REGISTRY, "/", tgz_relative_path]) + shasum = digest(path, HashType.SHA1) + dist = dict() + dist["tarball"] = tarball_link + dist["shasum"] = shasum + version_data["dist"] = dist + with open(version_meta_extract_path, mode='w', encoding='utf-8') as f: + dump(_del_none(version_data), f) + + +def __parse_npm_package_version_paths(path: str) -> Tuple[dict, list]: try: with open(path, encoding='utf-8') as version_package: data = load(version_package) package_version_paths = [data['name'], data['version']] - return package_version_paths + return data, package_version_paths except JSONDecodeError: logger.error('Error: Failed to parse json!') From 1758c372b232870a7cb92ac12425e70c1c179b1f Mon Sep 17 00:00:00 2001 From: yma Date: Sat, 2 Apr 2022 12:12:57 +0800 Subject: [PATCH 2/5] Add npm dist gen supporting for multi-targets and integrity computing * Add registry in target bucket configuration * Different registries dist-tarball gen for multi-targets * Tarball integrity computing for dist support * Add unit testings for dist gen computing against s3 --- charon/cmd/command.py | 5 +- charon/config.py | 18 ++++- charon/constants.py | 2 +- charon/pkgs/maven.py | 4 +- charon/pkgs/npm.py | 73 ++++++++--------- charon/utils/archive.py | 32 ++++---- charon/utils/map.py | 7 ++ requirements.txt | 1 + tests/base.py | 4 + tests/commons.py | 2 + tests/test_config.py | 17 ++++ tests/test_manifest_del.py | 9 ++- tests/test_manifest_upload.py | 5 +- tests/test_maven_del.py | 10 +-- tests/test_maven_del_multi_tgts.py | 6 +- tests/test_maven_index.py | 20 ++--- tests/test_maven_index_multi_tgts.py | 14 ++-- tests/test_maven_upload.py | 8 +- tests/test_maven_upload_multi_tgts.py | 15 ++-- tests/test_npm_del.py | 10 +-- tests/test_npm_del_multi_tgts.py | 7 +- tests/test_npm_dist_gen.py | 111 ++++++++++++++++++++++++++ tests/test_npm_index.py | 12 +-- tests/test_npm_index_multi_tgts.py | 10 ++- tests/test_npm_meta.py | 5 +- tests/test_npm_upload.py | 8 +- tests/test_npm_upload_multi_tgts.py | 8 +- tests/test_pkgs_dryrun.py | 17 ++-- 28 files changed, 297 insertions(+), 143 deletions(-) create mode 100644 charon/utils/map.py create mode 100644 tests/test_npm_dist_gen.py diff --git a/charon/cmd/command.py b/charon/cmd/command.py index d331038b..cbb4e110 100644 --- a/charon/cmd/command.py +++ b/charon/cmd/command.py @@ -345,14 +345,15 @@ def delete( __safe_delete(tmp_dir) -def __get_targets(target: List[str], conf: CharonConfig) -> List[Tuple[str, str, str]]: +def __get_targets(target: List[str], conf: CharonConfig) -> List[Tuple[str, str, str, str]]: targets_ = [] for tgt in target: aws_bucket = conf.get_aws_bucket(tgt) if not aws_bucket: continue prefix = conf.get_bucket_prefix(tgt) - targets_.append([tgt, aws_bucket, prefix]) + registry = conf.get_bucket_registry(tgt) + targets_.append([tgt, aws_bucket, prefix, registry]) if len(targets_) == 0: logger.error( "All targets are not valid or configured, " diff --git a/charon/config.py b/charon/config.py index c4324250..bb7308fa 100644 --- a/charon/config.py +++ b/charon/config.py @@ -20,6 +20,7 @@ import logging from charon.utils.strings import remove_prefix +from charon.constants import DEFAULT_REGISTRY CONFIG_FILE = "charon.yaml" @@ -53,8 +54,8 @@ def get_aws_bucket(self, target: str) -> str: return None bucket = target_.get("bucket", None) if not bucket: - logger.error("The bucket %s is not found for target %s " - "in charon configuration.") + logger.error("The bucket is not found for target %s " + "in charon configuration.", target) return bucket def get_bucket_prefix(self, target: str) -> str: @@ -73,6 +74,19 @@ def get_bucket_prefix(self, target: str) -> str: prefix = remove_prefix(prefix, "/") return prefix + def get_bucket_registry(self, target: str) -> str: + target_: Dict = self.__targets.get(target, None) + if not target_ or not isinstance(target_, Dict): + logger.error("The target %s is not found in charon configuration.", target) + return None + registry = target_.get("registry", None) + if not registry: + registry = DEFAULT_REGISTRY + logger.error("The registry is not found for target %s " + "in charon configuration, so DEFAULT_REGISTRY(localhost) will be used.", + target) + return registry + def get_manifest_bucket(self) -> str: return self.__manifest_bucket diff --git a/charon/constants.py b/charon/constants.py index 7b3e99a0..6751aecd 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -174,4 +174,4 @@ MANIFEST_SUFFIX = ".txt" DEFAULT_ERRORS_LOG = "errors.log" -NRRC_REGISTRY = "npm.registry.redhat.com" +DEFAULT_REGISTRY = "localhost" diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index edf29d45..cabc87ef 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -256,7 +256,7 @@ def handle_maven_uploading( prod_key: str, ignore_patterns=None, root="maven-repository", - targets: List[Tuple[str, str, str]] = None, + targets: List[Tuple[str, str, str, str]] = None, aws_profile=None, dir_=None, do_index=True, @@ -418,7 +418,7 @@ def handle_maven_del( prod_key: str, ignore_patterns=None, root="maven-repository", - targets: List[Tuple[str, str, str]] = None, + targets: List[Tuple[str, str, str, str]] = None, aws_profile=None, dir_=None, do_index=True, diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 2d8b8d87..8942c0c4 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -30,6 +30,7 @@ from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process from charon.utils.strings import remove_prefix from charon.utils.files import write_manifest +from charon.utils.map import del_none logger = logging.getLogger(__name__) @@ -65,7 +66,7 @@ def __init__(self, metadata, is_version): def handle_npm_uploading( tarball_path: str, product: str, - targets: List[Tuple[str, str, str]] = None, + targets: List[Tuple[str, str, str, str]] = None, aws_profile=None, dir_=None, do_index=True, @@ -86,48 +87,47 @@ def handle_npm_uploading( Returns the directory used for archive processing and if uploading is successful """ - target_dir, valid_paths, package_metadata = _scan_metadata_paths_from_archive( - tarball_path, prod=product, dir__=dir_ - ) - if not os.path.isdir(target_dir): - logger.error("Error: the extracted target_dir path %s does not exist.", target_dir) - sys.exit(1) + for target in targets: + bucket_ = target[1] + prefix__ = remove_prefix(target[2], "/") + registry__ = target[3] + target_dir, valid_paths, package_metadata = _scan_metadata_paths_from_archive( + tarball_path, registry__, prod=product, dir__=dir_ + ) + if not os.path.isdir(target_dir): + logger.error("Error: the extracted target_dir path %s does not exist.", target_dir) + sys.exit(1) + valid_dirs = __get_path_tree(valid_paths, target_dir) + + # main_target = targets[0] + client = S3Client(aws_profile=aws_profile, dry_run=dry_run) + logger.info("Start uploading files to s3 buckets: %s", bucket_) + failed_files = client.upload_files( + file_paths=valid_paths, + targets=[(bucket_, prefix__)], + product=product, + root=target_dir + ) - valid_dirs = __get_path_tree(valid_paths, target_dir) + logger.info("Files uploading done\n") - # main_target = targets[0] - client = S3Client(aws_profile=aws_profile, dry_run=dry_run) - targets_ = [(target[1], remove_prefix(target[2], "/")) for target in targets] - logger.info( - "Start uploading files to s3 buckets: %s", - [target[1] for target in targets] - ) - failed_files = client.upload_files( - file_paths=valid_paths, - targets=targets_, - product=product, - root=target_dir - ) - logger.info("Files uploading done\n") + succeeded = True - succeeded = True - for target in targets: if not manifest_bucket_name: logger.warning( 'Warning: No manifest bucket is provided, will ignore the process of manifest ' 'uploading\n') else: logger.info("Start uploading manifest to s3 bucket %s", manifest_bucket_name) - manifest_folder = target[1] + manifest_folder = bucket_ manifest_name, manifest_full_path = write_manifest(valid_paths, target_dir, product) + client.upload_manifest( manifest_name, manifest_full_path, manifest_folder, manifest_bucket_name ) logger.info("Manifest uploading is done\n") - bucket_ = target[1] - prefix__ = remove_prefix(target[2], "/") logger.info( "Start generating package.json for package: %s in s3 bucket %s", package_metadata.name, bucket_ @@ -178,7 +178,7 @@ def handle_npm_uploading( def handle_npm_del( tarball_path: str, product: str, - targets: List[Tuple[str, str, str]] = None, + targets: List[Tuple[str, str, str, str]] = None, aws_profile=None, dir_=None, do_index=True, @@ -381,11 +381,11 @@ def _gen_npm_package_metadata_for_del( return meta_files -def _scan_metadata_paths_from_archive(path: str, prod="", dir__=None) -> Tuple[str, list, - NPMPackageMetadata]: +def _scan_metadata_paths_from_archive(path: str, registry: str, prod="", dir__=None) ->\ + Tuple[str, list, NPMPackageMetadata]: tmp_root = mkdtemp(prefix=f"npm-charon-{prod}-", dir=dir__) try: - _, valid_paths = extract_npm_tarball(path, tmp_root, True) + _, valid_paths = extract_npm_tarball(path, tmp_root, True, registry) if len(valid_paths) > 1: version = _scan_for_version(valid_paths[1]) package = NPMPackageMetadata(version, True) @@ -502,7 +502,7 @@ def _write_package_metadata_to_file(package_metadata: NPMPackageMetadata, root=' final_package_metadata_path = os.path.join(root, package_metadata.name, PACKAGE_JSON) try: with open(final_package_metadata_path, mode='w', encoding='utf-8') as f: - dump(_del_none(package_metadata.__dict__.copy()), f) + dump(del_none(package_metadata.__dict__.copy()), f) return final_package_metadata_path except FileNotFoundError: logger.error( @@ -510,15 +510,6 @@ def _write_package_metadata_to_file(package_metadata: NPMPackageMetadata, root=' ) -def _del_none(d): - for key, value in list(d.items()): - if value is None: - del d[key] - elif isinstance(value, dict): - _del_none(value) - return d - - def __get_path_tree(paths: str, prefix: str) -> Set[str]: valid_dirs = set() for f in paths: diff --git a/charon/utils/archive.py b/charon/utils/archive.py index 7c485a19..5bcb2777 100644 --- a/charon/utils/archive.py +++ b/charon/utils/archive.py @@ -20,12 +20,14 @@ import requests import tempfile import shutil +import subresource_integrity from enum import Enum from json import load, JSONDecodeError, dump from typing import Tuple from zipfile import ZipFile, is_zipfile -from charon.constants import NRRC_REGISTRY +from charon.constants import DEFAULT_REGISTRY from charon.utils.files import digest, HashType +from charon.utils.map import del_none logger = logging.getLogger(__name__) @@ -44,7 +46,8 @@ def extract_zip_with_files(zf: ZipFile, target_dir: str, file_suffix: str, debug zf.extractall(target_dir, members=filtered) -def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool) -> Tuple[str, list]: +def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool, registry=DEFAULT_REGISTRY)\ + -> Tuple[str, list]: """ Extract npm tarball will relocate the tgz file and metadata files. * Locate tar path ( e.g.: jquery/-/jquery-7.6.1.tgz or @types/jquery/-/jquery-2.2.3.tgz). * Locate version metadata path (e.g.: jquery/7.6.1 or @types/jquery/2.2.3). @@ -68,7 +71,7 @@ def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool) -> Tupl if is_for_upload: tgz_relative_path = "/".join([parse_paths[0], "-", _get_tgz_name(path)]) - __write_npm_version_dist(path, f.path, version_data, tgz_relative_path) + __write_npm_version_dist(path, f.path, version_data, tgz_relative_path, registry) os.makedirs(tarball_parent_path) target = os.path.join(tarball_parent_path, os.path.basename(path)) @@ -87,25 +90,18 @@ def _get_tgz_name(path: str): return "" -def _del_none(d): - for key, value in list(d.items()): - if value is None: - del d[key] - elif isinstance(value, dict): - _del_none(value) - return d - - def __write_npm_version_dist(path: str, version_meta_extract_path: str, version_data: dict, - tgz_relative_path: str): - tarball_link = "".join(["https://", NRRC_REGISTRY, "/", tgz_relative_path]) - shasum = digest(path, HashType.SHA1) + tgz_relative_path: str, registry: str): dist = dict() - dist["tarball"] = tarball_link - dist["shasum"] = shasum + dist["tarball"] = "".join(["https://", registry, "/", tgz_relative_path]) + dist["shasum"] = digest(path, HashType.SHA1) + with open(path, "rb") as tarball: + tarball_data = tarball.read() + integrity = subresource_integrity.render(tarball_data, ['sha512']) + dist["integrity"] = integrity version_data["dist"] = dist with open(version_meta_extract_path, mode='w', encoding='utf-8') as f: - dump(_del_none(version_data), f) + dump(del_none(version_data), f) def __parse_npm_package_version_paths(path: str) -> Tuple[dict, list]: diff --git a/charon/utils/map.py b/charon/utils/map.py new file mode 100644 index 00000000..606b0d9e --- /dev/null +++ b/charon/utils/map.py @@ -0,0 +1,7 @@ +def del_none(d): + for key, value in list(d.items()): + if value is None: + del d[key] + elif isinstance(value, dict): + del_none(value) + return d diff --git a/requirements.txt b/requirements.txt index d35d6b0d..e63a211a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ click==8.0.3 requests==2.27.1 ruamel.yaml==0.17.20 defusedxml==0.7.1 +subresource-integrity==0.2 diff --git a/tests/base.py b/tests/base.py index 39995023..4e89c9ac 100644 --- a/tests/base.py +++ b/tests/base.py @@ -50,6 +50,10 @@ def setUp(self): ea: bucket: "charon-test-ea" prefix: earlyaccess/all + + npm: + bucket: "charon-test-npm" + registry: "npm1.registry.redhat.com" """ self.prepare_config(config_base, default_config_content) diff --git a/tests/commons.py b/tests/commons.py index 34f4e1d3..2e5f670b 100644 --- a/tests/commons.py +++ b/tests/commons.py @@ -101,6 +101,8 @@ "@babel/code-frame/-/code-frame-7.15.8.tgz", ] CODE_FRAME_META = "@babel/code-frame/package.json" + +CODE_FRAME_7_14_5_META = "@babel/code-frame/7.14.5/package.json" # For npm indexes CODE_FRAME_7_14_5_INDEXES = [ "@babel/code-frame/7.14.5/index.html", diff --git a/tests/test_config.py b/tests/test_config.py index 7014ed5d..41ce25ad 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -36,6 +36,7 @@ def test_config(self): self.assertEqual('ga', conf.get_bucket_prefix("ga")) self.assertEqual('charon-test-ea', conf.get_aws_bucket("ea")) self.assertEqual('earlyaccess/all', conf.get_bucket_prefix("ea")) + self.assertEqual('npm1.registry.redhat.com', conf.get_bucket_registry("npm")) def test_no_config(self): self.__base.change_home() @@ -84,6 +85,22 @@ def test_config_missing_prefix(self): self.assertEqual("charon-test", conf.get_aws_bucket("ga")) self.assertEqual("", conf.get_bucket_prefix("ga")) + def test_config_missing_registry(self): + content_missing_registry = """ +ignore_patterns: + - ".*^(redhat).*" + - ".*snapshot.*" + +targets: + npm: + bucket: charon-npm-test + """ + self.__change_config_content(content_missing_registry) + conf = config.get_config() + self.assertIsNotNone(conf) + self.assertEqual("charon-npm-test", conf.get_aws_bucket("npm")) + self.assertEqual("localhost", conf.get_bucket_registry("npm")) + def test_ignore_patterns(self): # pylint: disable=anomalous-backslash-in-string content_missing_targets = """ diff --git a/tests/test_manifest_del.py b/tests/test_manifest_del.py index c9a11c7f..bea22072 100644 --- a/tests/test_manifest_del.py +++ b/tests/test_manifest_del.py @@ -19,6 +19,7 @@ from charon.pkgs.maven import handle_maven_uploading, handle_maven_del from charon.pkgs.npm import handle_npm_uploading, handle_npm_del +from charon.constants import DEFAULT_REGISTRY from tests.base import PackageBaseTest from tests.commons import ( TEST_BUCKET, TEST_MANIFEST_BUCKET, TEST_TARGET, COMMONS_CLIENT_456_MANIFEST, @@ -41,7 +42,7 @@ def test_maven_manifest_delete(self): product = "commons-client-4.5.6" handle_maven_del( test_zip, product, - targets=[(TEST_TARGET, TEST_BUCKET, None)], + targets=[(TEST_TARGET, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -62,7 +63,7 @@ def test_npm_manifest_delete(self): product = "code-frame-7.14.5" handle_npm_del( test_tgz, product, - targets=[(TEST_TARGET, TEST_BUCKET, None)], + targets=[(TEST_TARGET, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -76,7 +77,7 @@ def __prepare_maven_content(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - targets=[(TEST_TARGET, TEST_BUCKET, None)], + targets=[(TEST_TARGET, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -87,7 +88,7 @@ def __prepare_npm_content(self): product = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product, - targets=[(TEST_TARGET, TEST_BUCKET, None)], + targets=[(TEST_TARGET, TEST_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET diff --git a/tests/test_manifest_upload.py b/tests/test_manifest_upload.py index 88541997..0f7251dd 100644 --- a/tests/test_manifest_upload.py +++ b/tests/test_manifest_upload.py @@ -19,6 +19,7 @@ from charon.pkgs.maven import handle_maven_uploading from charon.pkgs.npm import handle_npm_uploading +from charon.constants import DEFAULT_REGISTRY from tests.base import PackageBaseTest from tests.commons import ( TEST_BUCKET, TEST_MANIFEST_BUCKET, TEST_TARGET, COMMONS_CLIENT_456_MVN_NUM, @@ -35,7 +36,7 @@ def test_maven_manifest_upload(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - targets=[(TEST_TARGET, TEST_BUCKET, None)], + targets=[(TEST_TARGET, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -65,7 +66,7 @@ def test_npm_manifest_upload(self): product = "code-frame-7.14.5" handle_npm_uploading( test_zip, product, - targets=[(TEST_TARGET, TEST_BUCKET, None)], + targets=[(TEST_TARGET, TEST_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index f3d110be..712132ff 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -53,7 +53,7 @@ def test_ignore_del(self): handle_maven_del( test_zip, product_456, ignore_patterns=[".*.sha1"], - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False ) @@ -102,7 +102,7 @@ def __test_prefix_deletion(self, prefix: str): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir, do_index=False ) @@ -176,7 +176,7 @@ def __test_prefix_deletion(self, prefix: str): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir, do_index=False ) @@ -189,7 +189,7 @@ def __prepare_content(self, prefix=None): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir, do_index=False ) @@ -198,7 +198,7 @@ def __prepare_content(self, prefix=None): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_maven_del_multi_tgts.py b/tests/test_maven_del_multi_tgts.py index 931fff81..abf324bb 100644 --- a/tests/test_maven_del_multi_tgts.py +++ b/tests/test_maven_del_multi_tgts.py @@ -63,7 +63,7 @@ def test_ignore_del(self): handle_maven_del( test_zip, product_456, ignore_patterns=[".*.sha1"], - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False ) @@ -108,7 +108,7 @@ def test_ignore_del(self): def __test_prefix_deletion(self, prefix: str): self.__prepare_content(prefix) - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, None), (None, TEST_BUCKET_2, prefix, None)] test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_del( @@ -255,7 +255,7 @@ def __test_prefix_deletion(self, prefix: str): def __prepare_content(self, prefix=None): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, None), (None, TEST_BUCKET_2, prefix, None)] handle_maven_uploading( test_zip, product_456, targets=targets_, diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py index e55cfde6..bc8ccb9d 100644 --- a/tests/test_maven_index.py +++ b/tests/test_maven_index.py @@ -35,7 +35,7 @@ def test_uploading_index(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -79,7 +79,7 @@ def test_overlap_upload_index(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -87,7 +87,7 @@ def test_overlap_upload_index(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -139,7 +139,7 @@ def __test_upload_index_with_prefix(self, prefix: str): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir ) @@ -191,7 +191,7 @@ def test_deletion_index(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -239,7 +239,7 @@ def test_deletion_index(self): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -262,7 +262,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir ) @@ -309,7 +309,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir ) @@ -321,7 +321,7 @@ def __prepare_content(self, prefix=None): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir ) @@ -329,6 +329,6 @@ def __prepare_content(self, prefix=None): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir ) diff --git a/tests/test_maven_index_multi_tgts.py b/tests/test_maven_index_multi_tgts.py index 3c868479..47eb079d 100644 --- a/tests/test_maven_index_multi_tgts.py +++ b/tests/test_maven_index_multi_tgts.py @@ -40,7 +40,7 @@ def tearDown(self): super().tearDown() def test_uploading_index(self): - targets_ = [(None, TEST_BUCKET, None), (None, TEST_BUCKET_2, None)] + targets_ = [(None, TEST_BUCKET, None, None), (None, TEST_BUCKET_2, None, None)] test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( @@ -100,7 +100,7 @@ def test_uploading_index(self): self.assertNotIn(PROD_INFO_SUFFIX, index_content, msg=f'{bucket_name}') def test_overlap_upload_index(self): - targets_ = [(None, TEST_BUCKET, None), (None, TEST_BUCKET_2, None)] + targets_ = [(None, TEST_BUCKET, None, None), (None, TEST_BUCKET_2, None, None)] test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( @@ -188,7 +188,7 @@ def test_upload_index_with_root_prefix(self): self.__test_upload_index_with_prefix("/") def __test_upload_index_with_prefix(self, prefix: str): - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, None), (None, TEST_BUCKET_2, prefix, None)] test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( @@ -260,7 +260,7 @@ def test_deletion_index(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -308,7 +308,7 @@ def test_deletion_index(self): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -326,7 +326,7 @@ def test_deletion_index_with_root_prefix(self): def __test_deletion_index_with_prefix(self, prefix: str): self.__prepare_content(prefix) - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, None), (None, TEST_BUCKET_2, prefix, None)] test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_del( @@ -411,7 +411,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): self.assertEqual(0, len(objs), msg=f'{bucket_name}') def __prepare_content(self, prefix=None): - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, None), (None, TEST_BUCKET_2, prefix, None)] test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index c8424a10..b6165850 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -46,7 +46,7 @@ def test_overlap_upload(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False ) @@ -54,7 +54,7 @@ def test_overlap_upload(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False ) @@ -113,7 +113,7 @@ def test_ignore_upload(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, [".*.sha1"], - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, do_index=False ) @@ -142,7 +142,7 @@ def __test_prefix_upload(self, prefix: str): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_maven_upload_multi_tgts.py b/tests/test_maven_upload_multi_tgts.py index 90ca9102..85aea608 100644 --- a/tests/test_maven_upload_multi_tgts.py +++ b/tests/test_maven_upload_multi_tgts.py @@ -42,27 +42,28 @@ def tearDown(self): def test_fresh_upload(self): self.__test_prefix_upload( - [(None, TEST_BUCKET, ""), (None, TEST_BUCKET_2, "")] + [(None, TEST_BUCKET, ""), (None, TEST_BUCKET_2, "", None)] ) def test_short_prefix_upload(self): self.__test_prefix_upload( - [(None, TEST_BUCKET, SHORT_TEST_PREFIX), (None, TEST_BUCKET_2, SHORT_TEST_PREFIX)] + [(None, TEST_BUCKET, SHORT_TEST_PREFIX), (None, TEST_BUCKET_2, SHORT_TEST_PREFIX, None)] ) def test_long_prefix_upload(self): self.__test_prefix_upload( - [(None, TEST_BUCKET, LONG_TEST_PREFIX), (None, TEST_BUCKET_2, LONG_TEST_PREFIX)] + [(None, TEST_BUCKET, LONG_TEST_PREFIX), (None, TEST_BUCKET_2, LONG_TEST_PREFIX, None)] ) def test_root_prefix_upload(self): - self.__test_prefix_upload([(None, TEST_BUCKET, "/"), (None, TEST_BUCKET_2, "/")]) + self.__test_prefix_upload([(None, TEST_BUCKET, "/", None), + (None, TEST_BUCKET_2, "/", None)]) def test_overlap_upload(self): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" targets_ = [ - (None, TEST_BUCKET, None), (None, TEST_BUCKET_2, None) + (None, TEST_BUCKET, None, None), (None, TEST_BUCKET_2, None, None) ] handle_maven_uploading( test_zip, product_456, @@ -180,7 +181,7 @@ def test_ignore_upload(self): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" targets_ = [ - (None, TEST_BUCKET, None), (None, TEST_BUCKET_2, None) + (None, TEST_BUCKET, None, None), (None, TEST_BUCKET_2, None, None) ] handle_maven_uploading( test_zip, product_456, [".*.sha1"], @@ -214,7 +215,7 @@ def test_ignore_upload(self): for f in ignored_files: self.assertNotIn(f, actual_files, msg=f'{bucket_name}') - def __test_prefix_upload(self, targets: List[Tuple[str, str, str]]): + def __test_prefix_upload(self, targets: List[Tuple[str, str, str, str]]): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( diff --git a/tests/test_npm_del.py b/tests/test_npm_del.py index 5b37aefc..b2b1e332 100644 --- a/tests/test_npm_del.py +++ b/tests/test_npm_del.py @@ -15,7 +15,7 @@ """ import os from moto import mock_s3 -from charon.constants import PROD_INFO_SUFFIX +from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from charon.pkgs.npm import handle_npm_uploading, handle_npm_del from charon.storage import CHECKSUM_META_KEY from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, PackageBaseTest @@ -43,7 +43,7 @@ def __test_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir, do_index=False ) @@ -87,7 +87,7 @@ def __test_prefix(self, prefix: str = None): test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.15.8.tgz") handle_npm_del( test_tgz, product_7_15_8, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir, do_index=False ) objs = list(test_bucket.objects.all()) @@ -98,7 +98,7 @@ def __prepare_content(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) @@ -106,6 +106,6 @@ def __prepare_content(self, prefix: str = None): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_npm_del_multi_tgts.py b/tests/test_npm_del_multi_tgts.py index 82bfd85c..918d168a 100644 --- a/tests/test_npm_del_multi_tgts.py +++ b/tests/test_npm_del_multi_tgts.py @@ -15,7 +15,7 @@ """ import os from moto import mock_s3 -from charon.constants import PROD_INFO_SUFFIX +from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from charon.pkgs.npm import handle_npm_uploading, handle_npm_del from charon.storage import CHECKSUM_META_KEY from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, PackageBaseTest @@ -48,7 +48,7 @@ def test_npm_deletion_with_root_prefix(self): def __test_prefix(self, prefix: str = None): self.__prepare_content(prefix) - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, None), (None, TEST_BUCKET_2, prefix, None)] test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") product_7_14_5 = "code-frame-7.14.5" handle_npm_del( @@ -133,7 +133,8 @@ def __test_prefix(self, prefix: str = None): self.assertEqual(0, len(objs)) def __prepare_content(self, prefix: str = None): - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY), + (None, TEST_BUCKET_2, prefix, DEFAULT_REGISTRY)] test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( diff --git a/tests/test_npm_dist_gen.py b/tests/test_npm_dist_gen.py new file mode 100644 index 00000000..28d8b847 --- /dev/null +++ b/tests/test_npm_dist_gen.py @@ -0,0 +1,111 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import subresource_integrity +from moto import mock_s3 +from charon.pkgs.npm import handle_npm_uploading +from charon.utils.files import digest, HashType +from tests.base import PackageBaseTest +from tests.commons import ( + TEST_BUCKET, TEST_BUCKET_2, + CODE_FRAME_META, CODE_FRAME_7_14_5_META +) + + +@mock_s3 +class NPMUploadTest(PackageBaseTest): + def setUp(self): + super().setUp() + self.mock_s3.create_bucket(Bucket=TEST_BUCKET_2) + self.test_bucket_2 = self.mock_s3.Bucket(TEST_BUCKET_2) + + def test_dist_gen_in_single_target(self): + targets_ = [(None, TEST_BUCKET, None, "npm1.registry.redhat.com")] + test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") + product_7_14_5 = "code-frame-7.14.5" + handle_npm_uploading( + test_tgz, product_7_14_5, + targets=targets_, + dir_=self.tempdir, do_index=False + ) + test_bucket = self.mock_s3.Bucket(TEST_BUCKET) + meta_obj_client_7_14_5 = test_bucket.Object(CODE_FRAME_7_14_5_META) + meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", meta_content_client_7_14_5) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", meta_content_client_7_14_5) + self.assertIn("\"shasum\": \"23b08d740e83f49c5e59945fbf1b43e80bbf4edb\"", + meta_content_client_7_14_5) + self.assertIn("\"integrity\": " + "\"sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM" + "+vOtCS5ndmJicPJhKAwYRI6UfFw==\"", + meta_content_client_7_14_5) + + sha1 = digest(test_tgz, HashType.SHA1) + self.assertEqual(sha1, "23b08d740e83f49c5e59945fbf1b43e80bbf4edb") + with open(test_tgz, "rb") as tarball: + tarball_data = tarball.read() + sha512 = subresource_integrity.render(tarball_data, ['sha512']) + self.assertEqual(sha512, + "sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM" + "+vOtCS5ndmJicPJhKAwYRI6UfFw==") + + merged_meta_obj_client = test_bucket.Object(CODE_FRAME_META) + merged_meta_content_client = str(merged_meta_obj_client.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", merged_meta_content_client) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", merged_meta_content_client) + self.assertIn("\"shasum\": \"23b08d740e83f49c5e59945fbf1b43e80bbf4edb\"", + merged_meta_content_client) + self.assertIn("\"integrity\": " + "\"sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM" + "+vOtCS5ndmJicPJhKAwYRI6UfFw==\"", merged_meta_content_client) + + def test_dist_gen_in_multi_targets(self): + targets_ = [(None, TEST_BUCKET, None, "npm1.registry.redhat.com"), + (None, TEST_BUCKET_2, None, "npm2.registry.redhat.com")] + test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") + product_7_14_5 = "code-frame-7.14.5" + handle_npm_uploading( + test_tgz, product_7_14_5, + targets=targets_, + dir_=self.tempdir, do_index=False + ) + test_bucket_1 = self.mock_s3.Bucket(TEST_BUCKET) + meta_obj_client_7_14_5 = test_bucket_1.Object(CODE_FRAME_7_14_5_META) + meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", meta_content_client_7_14_5) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", meta_content_client_7_14_5) + + merged_meta_obj_client_1 = test_bucket_1.Object(CODE_FRAME_META) + merged_meta_content_client = str(merged_meta_obj_client_1.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", merged_meta_content_client) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", merged_meta_content_client) + + test_bucket_2 = self.mock_s3.Bucket(TEST_BUCKET_2) + meta_obj_client_7_14_5 = test_bucket_2.Object(CODE_FRAME_7_14_5_META) + meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", meta_content_client_7_14_5) + self.assertIn("\"tarball\": \"https://npm2.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", meta_content_client_7_14_5) + + merged_meta_obj_client_2 = test_bucket_2.Object(CODE_FRAME_META) + merged_meta_content_client = str(merged_meta_obj_client_2.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", merged_meta_content_client) + self.assertIn("\"tarball\": \"https://npm2.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", merged_meta_content_client) diff --git a/tests/test_npm_index.py b/tests/test_npm_index.py index c76676e9..31a0e71c 100644 --- a/tests/test_npm_index.py +++ b/tests/test_npm_index.py @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from charon.constants import PROD_INFO_SUFFIX +from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from charon.pkgs.npm import handle_npm_uploading, handle_npm_del from charon.storage import CHECKSUM_META_KEY from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, PackageBaseTest @@ -46,7 +46,7 @@ def __test_upload_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, ) @@ -126,7 +126,7 @@ def __test_deletion_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir ) @@ -157,7 +157,7 @@ def __test_deletion_prefix(self, prefix: str = None): test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.15.8.tgz") handle_npm_del( test_tgz, product_7_15_8, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, None)], dir_=self.tempdir ) @@ -169,7 +169,7 @@ def __prepare_content(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir ) @@ -177,6 +177,6 @@ def __prepare_content(self, prefix: str = None): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir ) diff --git a/tests/test_npm_index_multi_tgts.py b/tests/test_npm_index_multi_tgts.py index c54092d7..65f3e206 100644 --- a/tests/test_npm_index_multi_tgts.py +++ b/tests/test_npm_index_multi_tgts.py @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from charon.constants import PROD_INFO_SUFFIX +from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from charon.pkgs.npm import handle_npm_uploading, handle_npm_del from charon.storage import CHECKSUM_META_KEY from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, PackageBaseTest @@ -53,7 +53,8 @@ def test_uploding_index_with_root_prefix(self): self.__test_upload_prefix("/") def __test_upload_prefix(self, prefix: str = None): - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY), + (None, TEST_BUCKET_2, prefix, DEFAULT_REGISTRY)] test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( @@ -161,7 +162,7 @@ def test_deletion_index_with_root_prefix(self): def __test_deletion_prefix(self, prefix: str = None): self.__prepare_content(prefix) - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, None), (None, TEST_BUCKET_2, prefix, None)] test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") product_7_14_5 = "code-frame-7.14.5" handle_npm_del( @@ -219,7 +220,8 @@ def __test_deletion_prefix(self, prefix: str = None): self.assertEqual(0, len(objs), msg=f'{bucket_name}') def __prepare_content(self, prefix: str = None): - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY), + (None, TEST_BUCKET_2, prefix, DEFAULT_REGISTRY)] test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( diff --git a/tests/test_npm_meta.py b/tests/test_npm_meta.py index fb510ae5..6737a9e4 100644 --- a/tests/test_npm_meta.py +++ b/tests/test_npm_meta.py @@ -20,6 +20,7 @@ from charon.pkgs.npm import handle_npm_uploading, read_package_metadata_from_content from charon.storage import S3Client +from charon.constants import DEFAULT_REGISTRY from tests.base import BaseTest MY_BUCKET = "npm_bucket" @@ -68,7 +69,7 @@ def test_handle_npm_uploading_for_old_version(self): ) handle_npm_uploading( tarball_test_path, "kogito-tooling-workspace-0.9.0-3", - targets=[(None, MY_BUCKET, None)], + targets=[(None, MY_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir ) (files, _) = self.s3_client.get_files( @@ -121,7 +122,7 @@ def test_handle_npm_uploading_for_new_version(self): ) handle_npm_uploading( tarball_test_path, "kogito-tooling-workspace-0.9.0-3", - targets=[(None, MY_BUCKET, None)], + targets=[(None, MY_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir ) (files, _) = self.s3_client.get_files( diff --git a/tests/test_npm_upload.py b/tests/test_npm_upload.py index 399a04c5..9bbbb861 100644 --- a/tests/test_npm_upload.py +++ b/tests/test_npm_upload.py @@ -20,7 +20,7 @@ from charon.pkgs.npm import handle_npm_uploading from charon.pkgs.pkg_utils import is_metadata from charon.storage import CHECKSUM_META_KEY -from charon.constants import PROD_INFO_SUFFIX +from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, PackageBaseTest from tests.commons import ( TEST_BUCKET, CODE_FRAME_7_14_5_FILES, @@ -48,14 +48,14 @@ def test_double_uploads(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.15.8.tgz") product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_BUCKET) @@ -92,7 +92,7 @@ def __test_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, prefix)], + targets=[(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_npm_upload_multi_tgts.py b/tests/test_npm_upload_multi_tgts.py index 6fb049ce..3a3b7aa5 100644 --- a/tests/test_npm_upload_multi_tgts.py +++ b/tests/test_npm_upload_multi_tgts.py @@ -20,7 +20,7 @@ from charon.pkgs.npm import handle_npm_uploading from charon.pkgs.pkg_utils import is_metadata from charon.storage import CHECKSUM_META_KEY -from charon.constants import PROD_INFO_SUFFIX +from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, PackageBaseTest from tests.commons import ( TEST_BUCKET, CODE_FRAME_7_14_5_FILES, @@ -53,7 +53,8 @@ def test_upload_with_root_prefix(self): self.__test_prefix("/") def test_double_uploads(self): - targets_ = [(None, TEST_BUCKET, None), (None, TEST_BUCKET_2, None)] + targets_ = [(None, TEST_BUCKET, None, DEFAULT_REGISTRY), + (None, TEST_BUCKET_2, None, DEFAULT_REGISTRY)] test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( @@ -123,7 +124,8 @@ def test_double_uploads(self): ) def __test_prefix(self, prefix: str = None): - targets_ = [(None, TEST_BUCKET, prefix), (None, TEST_BUCKET_2, prefix)] + targets_ = [(None, TEST_BUCKET, prefix, DEFAULT_REGISTRY), + (None, TEST_BUCKET_2, prefix, DEFAULT_REGISTRY)] test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py index e75b0258..8eff2b36 100644 --- a/tests/test_pkgs_dryrun.py +++ b/tests/test_pkgs_dryrun.py @@ -15,6 +15,7 @@ """ from charon.pkgs.maven import handle_maven_uploading, handle_maven_del from charon.pkgs.npm import handle_npm_uploading, handle_npm_del +from charon.constants import DEFAULT_REGISTRY from tests.base import PackageBaseTest from tests.commons import TEST_BUCKET from moto import mock_s3 @@ -28,7 +29,7 @@ def test_maven_upload_dry_run(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, dry_run=True ) @@ -44,7 +45,7 @@ def test_maven_delete_dry_run(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, dry_run=True ) @@ -58,7 +59,7 @@ def test_npm_upload_dry_run(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir, dry_run=True ) @@ -74,7 +75,7 @@ def test_npm_deletion_dry_run(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir, dry_run=True ) @@ -88,7 +89,7 @@ def __prepare_maven_content(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -96,7 +97,7 @@ def __prepare_maven_content(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, None)], dir_=self.tempdir ) @@ -105,7 +106,7 @@ def __prepare_npm_content(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir ) @@ -113,6 +114,6 @@ def __prepare_npm_content(self): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - targets=[(None, TEST_BUCKET, None)], + targets=[(None, TEST_BUCKET, None, DEFAULT_REGISTRY)], dir_=self.tempdir ) From 9d08996ed886bcc8924e63ba3e24526e741c749b Mon Sep 17 00:00:00 2001 From: yma Date: Sat, 2 Apr 2022 15:35:44 +0800 Subject: [PATCH 3/5] Fix to avoid initiating s3 client repeatedly --- charon/pkgs/npm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 8942c0c4..9b4683fa 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -87,6 +87,7 @@ def handle_npm_uploading( Returns the directory used for archive processing and if uploading is successful """ + client = S3Client(aws_profile=aws_profile, dry_run=dry_run) for target in targets: bucket_ = target[1] prefix__ = remove_prefix(target[2], "/") @@ -99,8 +100,6 @@ def handle_npm_uploading( sys.exit(1) valid_dirs = __get_path_tree(valid_paths, target_dir) - # main_target = targets[0] - client = S3Client(aws_profile=aws_profile, dry_run=dry_run) logger.info("Start uploading files to s3 buckets: %s", bucket_) failed_files = client.upload_files( file_paths=valid_paths, @@ -108,7 +107,6 @@ def handle_npm_uploading( product=product, root=target_dir ) - logger.info("Files uploading done\n") succeeded = True From 2b6c535f13e4207f3ec1289dd77b7a2a6e5ccd3a Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 11 Apr 2022 09:58:24 +0800 Subject: [PATCH 4/5] Convert npm version-level package.json as metadata partially --- charon/pkgs/npm.py | 18 ++++++++++++++-- charon/storage.py | 21 +++++++++++-------- tests/test_npm_dist_gen.py | 43 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 9b4683fa..70c45a77 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -102,7 +102,7 @@ def handle_npm_uploading( logger.info("Start uploading files to s3 buckets: %s", bucket_) failed_files = client.upload_files( - file_paths=valid_paths, + file_paths=[valid_paths[0]], targets=[(bucket_, prefix__)], product=product, root=target_dir @@ -126,6 +126,21 @@ def handle_npm_uploading( ) logger.info("Manifest uploading is done\n") + logger.info( + "Start generating version-level package.json for package: %s in s3 bucket %s", + package_metadata.name, bucket_ + ) + failed_metas = [] + _version_metadata_path = valid_paths[1] + _failed_metas = client.upload_metadatas( + meta_file_paths=[_version_metadata_path], + target=(bucket_, prefix__), + product=product, + root=target_dir + ) + failed_metas.extend(_failed_metas) + logger.info("version-level package.json uploading done") + logger.info( "Start generating package.json for package: %s in s3 bucket %s", package_metadata.name, bucket_ @@ -135,7 +150,6 @@ def handle_npm_uploading( ) logger.info("package.json generation done\n") - failed_metas = [] if META_FILE_GEN_KEY in meta_files: _failed_metas = client.upload_metadatas( meta_file_paths=[meta_files[META_FILE_GEN_KEY]], diff --git a/charon/storage.py b/charon/storage.py index 71290711..d6177d03 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -383,15 +383,18 @@ async def path_upload_handler( # NOTE: This should not happen for most cases, as most # of the metadata file does not have product info. Just # leave for requirement change in future - (prods, no_error) = await self.__run_async( - self.__get_prod_info, - path_key, bucket_name - ) - if not no_error: - failed.append(full_file_path) - return - if no_error and product not in prods: - prods.append(product) + # This is now used for npm version-level package.json + prods = [product] + if existed: + (prods, no_error) = await self.__run_async( + self.__get_prod_info, + path_key, bucket_name + ) + if not no_error: + failed.append(full_file_path) + return + if no_error and product not in prods: + prods.append(product) updated = await self.__update_prod_info( path_key, bucket_name, prods ) diff --git a/tests/test_npm_dist_gen.py b/tests/test_npm_dist_gen.py index 28d8b847..e7721e41 100644 --- a/tests/test_npm_dist_gen.py +++ b/tests/test_npm_dist_gen.py @@ -109,3 +109,46 @@ def test_dist_gen_in_multi_targets(self): self.assertIn("\"dist\"", merged_meta_content_client) self.assertIn("\"tarball\": \"https://npm2.registry.redhat.com/@babel/code-frame/-/code" "-frame-7.14.5.tgz\"", merged_meta_content_client) + + def test_overlapping_registry_dist_gen(self): + targets_ = [(None, TEST_BUCKET, None, "npm1.registry.redhat.com")] + test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") + product_7_14_5 = "code-frame-7.14.5" + handle_npm_uploading( + test_tgz, product_7_14_5, + targets=targets_, + dir_=self.tempdir, do_index=False + ) + test_bucket = self.mock_s3.Bucket(TEST_BUCKET) + meta_obj_client_7_14_5 = test_bucket.Object(CODE_FRAME_7_14_5_META) + meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", meta_content_client_7_14_5) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", meta_content_client_7_14_5) + + merged_meta_obj_client = test_bucket.Object(CODE_FRAME_META) + merged_meta_content_client = str(merged_meta_obj_client.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", merged_meta_content_client) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", merged_meta_content_client) + + targets_overlapping_ = [(None, TEST_BUCKET, None, "npm1.overlapping.registry.redhat.com")] + test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") + product_7_14_5 = "code-frame-7.14.5" + handle_npm_uploading( + test_tgz, product_7_14_5, + targets=targets_overlapping_, + dir_=self.tempdir, do_index=False + ) + + meta_obj_client_7_14_5 = test_bucket.Object(CODE_FRAME_7_14_5_META) + meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", meta_content_client_7_14_5) + self.assertIn("\"tarball\": \"https://npm1.overlapping.registry.redhat.com/@babel/code" + "-frame/-/code-frame-7.14.5.tgz\"", meta_content_client_7_14_5) + + merged_meta_obj_client = test_bucket.Object(CODE_FRAME_META) + merged_meta_content_client = str(merged_meta_obj_client.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", merged_meta_content_client) + self.assertIn("\"tarball\": \"https://npm1.overlapping.registry.redhat.com/@babel/code" + "-frame/-/code-frame-7.14.5.tgz\"", merged_meta_content_client) From 21236e1b46b0d1451814590f5256afaaae2f6395 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 11 Apr 2022 16:11:15 +0800 Subject: [PATCH 5/5] Update version to 1.1.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9efc9785..f26e196e 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup, find_packages -version = "1.1.0" +version = "1.1.1" # f = open('README.md') # long_description = f.read().strip()