diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 9b4683fa..70c45a77 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -102,7 +102,7 @@ def handle_npm_uploading( logger.info("Start uploading files to s3 buckets: %s", bucket_) failed_files = client.upload_files( - file_paths=valid_paths, + file_paths=[valid_paths[0]], targets=[(bucket_, prefix__)], product=product, root=target_dir @@ -126,6 +126,21 @@ def handle_npm_uploading( ) logger.info("Manifest uploading is done\n") + logger.info( + "Start generating version-level package.json for package: %s in s3 bucket %s", + package_metadata.name, bucket_ + ) + failed_metas = [] + _version_metadata_path = valid_paths[1] + _failed_metas = client.upload_metadatas( + meta_file_paths=[_version_metadata_path], + target=(bucket_, prefix__), + product=product, + root=target_dir + ) + failed_metas.extend(_failed_metas) + logger.info("version-level package.json uploading done") + logger.info( "Start generating package.json for package: %s in s3 bucket %s", package_metadata.name, bucket_ @@ -135,7 +150,6 @@ def handle_npm_uploading( ) logger.info("package.json generation done\n") - failed_metas = [] if META_FILE_GEN_KEY in meta_files: _failed_metas = client.upload_metadatas( meta_file_paths=[meta_files[META_FILE_GEN_KEY]], diff --git a/charon/storage.py b/charon/storage.py index 71290711..d6177d03 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -383,15 +383,18 @@ async def path_upload_handler( # NOTE: This should not happen for most cases, as most # of the metadata file does not have product info. Just # leave for requirement change in future - (prods, no_error) = await self.__run_async( - self.__get_prod_info, - path_key, bucket_name - ) - if not no_error: - failed.append(full_file_path) - return - if no_error and product not in prods: - prods.append(product) + # This is now used for npm version-level package.json + prods = [product] + if existed: + (prods, no_error) = await self.__run_async( + self.__get_prod_info, + path_key, bucket_name + ) + if not no_error: + failed.append(full_file_path) + return + if no_error and product not in prods: + prods.append(product) updated = await self.__update_prod_info( path_key, bucket_name, prods ) diff --git a/tests/test_npm_dist_gen.py b/tests/test_npm_dist_gen.py index 28d8b847..e7721e41 100644 --- a/tests/test_npm_dist_gen.py +++ b/tests/test_npm_dist_gen.py @@ -109,3 +109,46 @@ def test_dist_gen_in_multi_targets(self): self.assertIn("\"dist\"", merged_meta_content_client) self.assertIn("\"tarball\": \"https://npm2.registry.redhat.com/@babel/code-frame/-/code" "-frame-7.14.5.tgz\"", merged_meta_content_client) + + def test_overlapping_registry_dist_gen(self): + targets_ = [(None, TEST_BUCKET, None, "npm1.registry.redhat.com")] + test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") + product_7_14_5 = "code-frame-7.14.5" + handle_npm_uploading( + test_tgz, product_7_14_5, + targets=targets_, + dir_=self.tempdir, do_index=False + ) + test_bucket = self.mock_s3.Bucket(TEST_BUCKET) + meta_obj_client_7_14_5 = test_bucket.Object(CODE_FRAME_7_14_5_META) + meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", meta_content_client_7_14_5) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", meta_content_client_7_14_5) + + merged_meta_obj_client = test_bucket.Object(CODE_FRAME_META) + merged_meta_content_client = str(merged_meta_obj_client.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", merged_meta_content_client) + self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code" + "-frame-7.14.5.tgz\"", merged_meta_content_client) + + targets_overlapping_ = [(None, TEST_BUCKET, None, "npm1.overlapping.registry.redhat.com")] + test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") + product_7_14_5 = "code-frame-7.14.5" + handle_npm_uploading( + test_tgz, product_7_14_5, + targets=targets_overlapping_, + dir_=self.tempdir, do_index=False + ) + + meta_obj_client_7_14_5 = test_bucket.Object(CODE_FRAME_7_14_5_META) + meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", meta_content_client_7_14_5) + self.assertIn("\"tarball\": \"https://npm1.overlapping.registry.redhat.com/@babel/code" + "-frame/-/code-frame-7.14.5.tgz\"", meta_content_client_7_14_5) + + merged_meta_obj_client = test_bucket.Object(CODE_FRAME_META) + merged_meta_content_client = str(merged_meta_obj_client.get()["Body"].read(), "utf-8") + self.assertIn("\"dist\"", merged_meta_content_client) + self.assertIn("\"tarball\": \"https://npm1.overlapping.registry.redhat.com/@babel/code" + "-frame/-/code-frame-7.14.5.tgz\"", merged_meta_content_client)