From 93184116a90162fd9034ea1ea23cffbaeab7614a Mon Sep 17 00:00:00 2001 From: John Casey Date: Mon, 6 Dec 2021 17:06:33 -0600 Subject: [PATCH 1/3] Feat: Maintain archetype-catalog.xml for Maven Feat: Maintain sha1, md5, and sha256 files for maven-metadata.xml NOTE: The tests are VERY broken currently. I haven't figured out why yet. --- charon/constants.py | 14 ++ charon/pkgs/maven.py | 335 +++++++++++++++++++++++++-- charon/pkgs/npm.py | 3 +- charon/storage.py | 35 +-- requirements.txt | 1 + template/archetype-catalog.xml.j2 | 11 + tests/commons.py | 8 + tests/input/commons-client-4.5.6.zip | Bin 14677 -> 15062 bytes tests/input/commons-client-4.5.9.zip | Bin 13951 -> 14337 bytes tests/test_maven_del.py | 15 +- tests/test_maven_upload.py | 54 +++-- 11 files changed, 420 insertions(+), 56 deletions(-) create mode 100644 template/archetype-catalog.xml.j2 diff --git a/charon/constants.py b/charon/constants.py index 80d2c8f8..18586eb5 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -13,6 +13,20 @@ See the License for the specific language governing permissions and limitations under the License. """ +ARCHETYPE_CATALOG_FILENAME = "archetype-catalog.xml" +ARCHETYPE_CATALOG_TEMPLATE = ''' + + + {% for arch in archetypes %} + + {{ arch.group_id }} + {{ arch.artifact_id }} + {{ arch.version }} + {{ arch.description }} + {% endfor %} + + +''' # Logging format used CHARON_LOGGING_FMT = '%(asctime)s - %(levelname)s - %(message)s' DESCRIPTION = "charon is a tool to synchronize several types of artifacts " diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 6cf9712b..063fd39b 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -22,12 +22,15 @@ from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process from charon.config import get_template from charon.constants import (META_FILE_GEN_KEY, META_FILE_DEL_KEY, - META_FILE_FAILED, MAVEN_METADATA_TEMPLATE) -from typing import Dict, List, Tuple + META_FILE_FAILED, MAVEN_METADATA_TEMPLATE, + ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME) +from typing import Dict, List, Tuple, Optional from jinja2 import Template from datetime import datetime from zipfile import ZipFile, BadZipFile from tempfile import mkdtemp +from defusedxml import ElementTree + import os import sys import logging @@ -37,17 +40,19 @@ logger = logging.getLogger(__name__) -def __get_mvn_template() -> str: - """Gets the jinja2 template file content for maven-metadata.xml generation""" +def __get_mvn_template(kind: str, default: str) -> str: + """Gets the jinja2 template file content for metadata generation""" try: - return get_template("maven-metadata.xml.j2") + return get_template(kind) except FileNotFoundError: - logger.info("maven-metadata.xml template file not defined," - " will use default template.") - return MAVEN_METADATA_TEMPLATE + logger.info("%s template file not defined," + " will use default template.", kind) + return default -META_TEMPLATE = __get_mvn_template() +META_TEMPLATE = __get_mvn_template("maven-metadata.xml.j2", MAVEN_METADATA_TEMPLATE) +ARCH_TEMPLATE = __get_mvn_template("archetype-catalog.xml.j2", ARCHETYPE_CATALOG_TEMPLATE) +STANDARD_GENERATED_IGNORES = ["maven-metadata.xml", "archetype-catalog.xml"] class MavenMetadata(object): @@ -85,6 +90,48 @@ def __str__(self) -> str: return f"{self.group_id}:{self.artifact_id}\n{self.versions}\n\n" +class ArchetypeRef(object): + """This ArchetypeRef will represent an entry in archetype-catalog.xml content which will be + used in jinja2 or other places + """ + + def __init__(self, group_id: str, artifact_id: str, version: str, description: str): + self.group_id = group_id + self.artifact_id = artifact_id + self.version = version + self.description = description + + def __hash__(self): + return hash(self.group_id + self.artifact_id + self.version) + + def __eq__(self, other) -> bool: + if isinstance(other, ArchetypeRef): + return self.group_id == other.group_id \ + and self.artifact_id == other.artifact_id \ + and self.version == other.version + + return False + + def __str__(self) -> str: + return f"{self.group_id}:{self.artifact_id}\n{self.version}\n{self.description}\n\n" + + +class MavenArchetypeCatalog(object): + """This MavenArchetypeCatalog represents an archetype-catalog.xml which will be + used in jinja2 to regenerate the file with merged contents + """ + + def __init__(self, archetypes: List[ArchetypeRef]): + self.archetypes = sorted(set(archetypes), key=ArchetypeCompareKey) + + def generate_meta_file_content(self) -> str: + template = Template(ARCHETYPE_CATALOG_TEMPLATE) + return template.render(meta=self) + + def __str__(self) -> str: + return f"(Archetype Catalog with {len(self.archetypes)} entries).\n\n" + + def scan_for_poms(full_path: str) -> List[str]: """Scan a file path and finds all pom files absolute paths""" # collect poms @@ -280,6 +327,29 @@ def handle_maven_uploading( failed_metas.extend(_failed_metas) logger.info("maven-metadata.xml updating done\n") + # 7. Determine refreshment of archetype-catalog.xml + if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")): + logger.info("Start generating archetype-catalog.xml") + upload_archetype_file = _generate_upload_archetype_catalog( + s3=s3_client, bucket=bucket, + root=top_level, + prefix=prefix_ + ) + logger.info("archetype-catalog.xml files generation done\n") + + # 8. Upload archetype-catalog.xml if it has changed + if upload_archetype_file: + logger.info("Start updating archetype-catalog.xml to s3") + (_, _failed_metas) = s3_client.upload_metadatas( + meta_file_paths=[os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)], + bucket_name=bucket, + product=None, + root=top_level, + key_prefix=prefix_ + ) + failed_metas.extend(_failed_metas) + logger.info("archetype-catalog.xml updating done\n") + # this step generates index.html for each dir and add them to file list # index is similar to metadata, it will be overwritten everytime if do_index: @@ -295,7 +365,8 @@ def handle_maven_uploading( bucket_name=bucket, product=None, root=top_level, - key_prefix=prefix_ + key_prefix=prefix_, + digests=False ) failed_metas.extend(_failed_metas) logger.info("Index files updating done\n") @@ -399,6 +470,38 @@ def handle_maven_del( failed_metas.extend(_failed_metas) logger.info("maven-metadata.xml updating done\n") + # 7. Determine refreshment of archetype-catalog.xml + if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")): + logger.info("Start generating archetype-catalog.xml") + archetype_action = _generate_rollback_archetype_catalog( + s3=s3_client, bucket=bucket, + root=top_level, + prefix=prefix_ + ) + logger.info("archetype-catalog.xml files generation done\n") + + # 8. Upload or Delete archetype-catalog.xml if it has changed + if archetype_action < 0: + logger.info("Start updating archetype-catalog.xml to s3") + (_, _failed_metas) = s3_client.delete_files( + file_paths=[os.path.join(root, ARCHETYPE_CATALOG_FILENAME)], + bucket_name=bucket, + product=None, + root=top_level, + key_prefix=prefix_ + ) + failed_metas.extend(_failed_metas) + elif archetype_action > 0: + (_, _failed_metas) = s3_client.upload_metadatas( + meta_file_paths=[os.path.join(root, ARCHETYPE_CATALOG_FILENAME)], + bucket_name=bucket, + product=None, + root=top_level, + key_prefix=prefix_ + ) + failed_metas.extend(_failed_metas) + logger.info("archetype-catalog.xml updating done\n") + if do_index: logger.info("Start generating index files for all changed entries") created_indexes = indexing.generate_indexes( @@ -459,16 +562,19 @@ def _scan_paths(files_root: str, ignore_patterns: List[str], for name in names: path = os.path.join(root_dir, name) - if _is_ignored(name, ignore_patterns): - ignored_paths.append(name) - continue if top_level in root_dir: + # Let's wait to do the regex / pom examination until we know we're inside a valid root directory. + if _is_ignored(name, ignore_patterns): + ignored_paths.append(name) + continue + valid_mvn_paths.append(path) + + if name.strip().endswith(".pom"): + logger.debug("Found pom %s", name) + valid_poms.append(path) else: non_mvn_paths.append(path) - if name.strip().endswith(".pom"): - logger.debug("Found pom %s", name) - valid_poms.append(path) if len(non_mvn_paths) > 0: non_mvn_items = [n.replace(files_root, "") for n in non_mvn_paths] @@ -497,6 +603,175 @@ def _scan_paths(files_root: str, ignore_patterns: List[str], return (top_level, valid_mvn_paths, valid_poms, valid_dirs) +def _generate_rollback_archetype_catalog( + s3: S3Client, bucket: str, + root: str, prefix: str = None +) -> int: + """Determine whether the local archive contains /archetype-catalog.xml in the repo contents. + If so, determine whether the archetype-catalog.xml is already available in the bucket. Merge (or unmerge) these + catalogs and return an integer, indicating whether the bucket file should be replaced (+1), deleted (-1), or, + in the case where no action is required, it will return NO-OP (0). + + NOTE: There are three return values: + - +1 - UPLOAD the local catalog with its rolled back changes + - -1 - DELETE the (now empty) bucket catalog + - 0 - take no action + """ + local = os.path.join(root, ARCHETYPE_CATALOG_FILENAME) + if prefix: + remote = os.path.join(prefix, ARCHETYPE_CATALOG_FILENAME) + else: + remote = ARCHETYPE_CATALOG_FILENAME + + do_write = False + + # If there is no local catalog, this is a NO-OP + if os.path.exists(local): + if not s3.file_exists_in_bucket(bucket, remote): + # If there is no catalog in the bucket...this is a NO-OP + return 0 + else: + # If there IS a catalog in the bucket, we need to merge or un-merge it. + with open(local) as f: + local_archetypes = _parse_archetypes(f.read()) + + if len(local_archetypes) < 1: + # If there are no local archetypes in the catalog, there's nothing to do. + logger.warning( + "No archetypes found in local archetype-catalog.xml, even though the file exists! Skipping." + ) + return 0 + + else: + # Read the archetypes from the bucket so we can do a merge / un-merge + remote_xml = s3.read_file_content(bucket, remote) + remote_archetypes = _parse_archetypes(remote_xml) + + if len(remote_archetypes) < 1: + # Nothing in the bucket. Clear out this empty file. + return -1 + + else: + # If we're deleting, un-merge the local archetypes from the remote ones. + # + # NOTE: The ONLY reason we can get away with this kind of naive un-merge is that products only + # bother to publish archetypes for their own direct users. If they publish an archetype, it's only + # for use with their product. Therefore, if we rollback that product, the archetypes they reference + # shouldn't be useful anymore. + for la in local_archetypes: + if la not in remote_archetypes: + remote_archetypes.remove(la) + + if len(remote_archetypes) < 1: + # If there are no remote archetypes left after removing ours DELETE the bucket catalog. + return -1 + else: + # Re-render the result of our archetype un-merge to the local file, in preparation for upload. + with open(local, 'w') as f: + content = MavenArchetypeCatalog(remote_archetypes).generate_meta_file_content() + try: + overwrite_file(local, content) + except FileNotFoundError as e: + logger.error( + "Error: Can not create file %s because of some missing folders", + local, + ) + raise e + + return 1 + + return 0 + + +def _generate_upload_archetype_catalog( + s3: S3Client, bucket: str, + root: str, prefix: str = None +) -> bool: + """Determine whether the local archive contains /archetype-catalog.xml in the repo contents. + If so, determine whether the archetype-catalog.xml is already available in the bucket. Merge (or unmerge) these + catalogs and return a boolean indicating whether the local file should be uploaded. + """ + local = os.path.join(root, ARCHETYPE_CATALOG_FILENAME) + if prefix: + remote = os.path.join(prefix, ARCHETYPE_CATALOG_FILENAME) + else: + remote = ARCHETYPE_CATALOG_FILENAME + + # If there is no local catalog, this is a NO-OP + if os.path.exists(local): + if not s3.file_exists_in_bucket(bucket, remote): + # If there is no catalog in the bucket, just upload what we have locally + return True + else: + # If there IS a catalog in the bucket, we need to merge or un-merge it. + with open(local) as f: + local_archetypes = _parse_archetypes(f.read()) + + if len(local_archetypes) < 1: + logger.warning( + "No archetypes found in local archetype-catalog.xml, even though the file exists! Skipping." + ) + + else: + # Read the archetypes from the bucket so we can do a merge / un-merge + remote_xml = s3.read_file_content(bucket, remote) + remote_archetypes = _parse_archetypes(remote_xml) + + if len(remote_archetypes) < 1: + # Nothing in the bucket. Just push what we have locally. + return True + else: + original_remote_size = len(remote_archetypes) + for la in local_archetypes: + # The cautious approach in this operation contradicts assumptions we make for the rollback case. + # That's because we should NEVER encounter a collision on archetype GAV...they should + # belong with specific product releases. + # + # Still, we will WARN, not ERROR if we encounter this. + if la not in remote_archetypes: + remote_archetypes.append(la) + else: + logger.warning( + "\n\n\nDUPLICATE ARCHETYPE: %s. " + "This makes rollback of the current release UNSAFE!\n\n\n", + la + ) + + if len(remote_archetypes) != original_remote_size: + # If the number of archetypes in the version of the file from the bucket has changed, we need + # to regenerate the file and re-upload it. + # + # Re-render the result of our archetype merge / un-merge to the local file, in preparation for + # upload. + with open(local, 'w') as f: + content = MavenArchetypeCatalog(remote_archetypes).generate_meta_file_content() + try: + overwrite_file(local, content) + except FileNotFoundError as e: + logger.error( + "Error: Can not create file %s because of some missing folders", + local, + ) + raise e + + return True + + return False + + +def _parse_archetypes(source) -> List[ArchetypeRef]: + tree = ElementTree.fromstring(source.strip(), forbid_dtd=True, forbid_entities=True, forbid_external=True) + archetypes = [] + for a in tree.findall("./archetypes/archetype"): + gid = a.find('groupId').text + aid = a.find('artifactId').text + ver = a.find('version').text + desc = a.find('description').text + archetypes.append(ArchetypeRef(gid, aid, ver, desc)) + + return archetypes + + def _generate_metadatas( s3: S3Client, bucket: str, poms: List[str], root: str, @@ -580,9 +855,11 @@ def __hash_decorate_metadata(path: str, metadata: str) -> List[str]: def _is_ignored(filename: str, ignore_patterns: List[str]) -> bool: - if "maven-metadata.xml" in filename: - logger.warning("Ignoring Maven metadata file from input: %s", filename) - return True + for ignored_name in STANDARD_GENERATED_IGNORES: + if ignored_name in filename: + logger.warning("Ignoring standard generated Maven path: %s", filename) + return True + if ignore_patterns: for dirs in ignore_patterns: if re.search(dirs, filename): @@ -659,3 +936,23 @@ def __compare(self, other) -> int: else: continue return 0 + + +class ArchetypeCompareKey(VersionCompareKey): + 'Used as key function for GAV sorting' + def __init__(self, gav): + super().__init__(gav.version) + self.gav = gav + + def __compare(self, other) -> int: + x = self.gav.group_id + ":" + self.gav.artifact_id + y = other.gav.group_id + ":" + other.gav.artifact_id + + if x == y: + return 0 + elif x < y: + return -1 + else: + return 1 + + diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 5b78a5e7..c4a7feb5 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -131,7 +131,8 @@ def handle_npm_uploading( (_, _failed_metas) = client.upload_metadatas( meta_file_paths=created_indexes, bucket_name=bucket, product=None, - root=target_dir, key_prefix=prefix_ + root=target_dir, key_prefix=prefix_, + digests=False ) failed_metas.extend(_failed_metas) logger.info("Index files updating done\n") diff --git a/charon/storage.py b/charon/storage.py index 976109f7..17fa13ac 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -140,7 +140,7 @@ def path_upload_handler(full_file_path: str, path: str) -> bool: full_file_path, ExtraArgs={'ContentType': content_type} ) - logger.info('Uploaded %s to bucket %s', full_file_path, bucket_name) + logger.info('Uploaded %s to bucket %s', path, bucket_name) uploaded_files.append(path_key) except (ClientError, HTTPClientError) as e: logger.error("ERROR: file %s not uploaded to bucket" @@ -208,16 +208,18 @@ def path_upload_handler(full_file_path: str, path: str): return False logger.info('Updating metadata %s to bucket %s', path, bucket_name) path_key = os.path.join(key_prefix, path) if key_prefix else path - fileObject = bucket.Object(path_key) - existed = self.file_exists(fileObject) + file_object = bucket.Object(path_key) + existed = self.file_exists(file_object) f_meta = {} need_overwritten = True + sha1 = read_sha1(full_file_path) + (content_type, _) = mimetypes.guess_type(full_file_path) if not content_type: content_type = DEFAULT_MIME_TYPE if existed: - f_meta = fileObject.metadata + f_meta = file_object.metadata need_overwritten = ( CHECKSUM_META_KEY not in f_meta or sha1 != f_meta[CHECKSUM_META_KEY] ) @@ -234,15 +236,16 @@ def path_upload_handler(full_file_path: str, path: str): try: if not self.dry_run: if need_overwritten: - fileObject.put( + file_object.put( Body=open(full_file_path, "rb"), Metadata=f_meta, ContentType=content_type ) + else: # Should we update the s3 object metadata for metadata files? try: - self.__update_file_metadata(fileObject, bucket_name, path_key, f_meta) + self.__update_file_metadata(file_object, bucket_name, path_key, f_meta) except (ClientError, HTTPClientError) as e: logger.error("ERROR: metadata %s not updated to bucket" " %s due to error: %s ", full_file_path, @@ -263,7 +266,7 @@ def path_upload_handler(full_file_path: str, path: str): def delete_files( self, file_paths: List[str], bucket_name: str, - product: str, root="/", key_prefix: str = None + product: Optional[str], root="/", key_prefix: str = None ) -> Tuple[List[str], List[str]]: """ Deletes a list of files to s3 bucket. * Use the cut down file path as s3 key. The cut down way is move root from the file path if it starts with root. Example: if file_path is @@ -285,13 +288,19 @@ def path_delete_handler(full_file_path: str, path: str): fileObject = bucket.Object(path_key) existed = self.file_exists(fileObject) if existed: + # NOTE: If we're NOT using the product key to track collisions (in the case of metadata), then + # This prods array will remain empty, and we will just delete the file, below. Otherwise, the product + # reference counts will be used (from object metadata). prods = [] - try: - prods = fileObject.metadata[PRODUCT_META_KEY].split(",") - except KeyError: - pass - if product and product in prods: - prods.remove(product) + if product: + try: + prods = fileObject.metadata[PRODUCT_META_KEY].split(",") + except KeyError: + pass + + if product in prods: + prods.remove(product) + if len(prods) > 0: try: logger.info( diff --git a/requirements.txt b/requirements.txt index 8f0aa0a7..14d787ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ botocore click requests ruamel.yaml +defusedxml diff --git a/template/archetype-catalog.xml.j2 b/template/archetype-catalog.xml.j2 new file mode 100644 index 00000000..fb6e3f64 --- /dev/null +++ b/template/archetype-catalog.xml.j2 @@ -0,0 +1,11 @@ + + + {% for arch in archetypes %} + + {{ arch.group_id }} + {{ arch.artifact_id }} + {{ arch.version }} + {{ arch.description }} + {% endfor %} + + diff --git a/tests/commons.py b/tests/commons.py index c5dd93c3..61a61c4a 100644 --- a/tests/commons.py +++ b/tests/commons.py @@ -37,6 +37,14 @@ "commons-logging/commons-logging/maven-metadata.xml.sha256" ] +ARCHETYPE_CATALOG = "archetype-catalog.xml" +ARCHETYPE_CATALOG_FILES = [ + ARCHETYPE_CATALOG, + "archetype-catalog.xml.sha1", + "archetype-catalog.xml.md5", + "archetype-catalog.xml.sha256" +] + NON_MVN_FILES = [ "commons-client-4.5.6/example-settings.xml", "commons-client-4.5.6/licenses/gnu", diff --git a/tests/input/commons-client-4.5.6.zip b/tests/input/commons-client-4.5.6.zip index 5522517ef657759d36e0965eb44680309c8cfbb4..7ce27974ab2b78614e460bb6cdd691b40359bfc7 100644 GIT binary patch delta 504 zcmcawbgguP4Rch_x{3Co%-$eMYvVy7M&=n!ZIiDt+OY5MS(gaJ6YZrYPiGRCEW;={ z`2dqBPiP1y12bD@Xd+k(NX`$U8=_2z*%!vGVa|XuH#4$SGQrr>*zuV_u0{{8>t9A>fkO@Az6Y`FA9?q)VAukm6wo24Lj>xZGW)ncgv$Pl-9_K0tr5DR`0keZlv$aj;l;+~8?3+O@m-yM=2BDosjq83FWT{P z)yAyjw?5u#{4%w%FaQ7KjhZr>`82;m0-H_Siy7$X$qRvk`P!FwCkGg^Oy<{PQ^4>! tg9I=PfS~}yOBz3EOkQZDA&Vm*SQ)@(>RU?i$uY<=oD*PR_@oKa3;;dmx5EGc delta 248 zcmcasdbMbR4O91&$%dxFllz%CCKoUYOw80~sz4Fhcu-3qzP9F*{Z=*Ez&1 U!pg<~1hPQrE5N|;Ndv?K0E5Ly0ssI2 diff --git a/tests/input/commons-client-4.5.9.zip b/tests/input/commons-client-4.5.9.zip index 02f3fa7503ac820cf38e8b6ebf88d85ee467a6a2..a42de5a5edb93fd19f115a5aca828afde63abb14 100644 GIT binary patch delta 544 zcmeyL(^xRUmZ@jmPG=IB zEW;={`2dqBPiP1y1GBe6NFq$*bS6KzAs~f9%)W4b4RZ#Z{YYDAvM@^wlrJ**fB@U( z9+tV>a0PAx$uP;y2L#qKf;gKCgno)l-Y38z2y|C)*6EDS9tH*mMn(n(>&c3|n)QiA z$r-67l?AD~$%!S2Ir-^&6}dTJCqmtySX#l&z{v7~nSlXJ`1U(;F&Xl>UVp5)r{S%e zxYLV!d&N8E-V!_JnPJ?t-CXxfz=>yn|0{4NeAGPAfB5L)2Tm8~s#zRbWs8LYKSoXS-ezN_E$lk{GV^8Axpjk1r{)jF@78CGs;Ud+Bb`Teti z+LJoE70Y7ITj_VrUMID>Y>n)CS&w+h7eBu7Pwv%_+1#xD6%y*znqJJH5MN*FSdUwKbe^gaj)C$V~K52=HcQ1DPNLgw6a6 J3>g|A9su3Q!;b&} delta 239 zcmZoH_@6VumTAuF$pu1ElM5IHCT40g$04{I4+=3(zQ!*w`7xtDdrVGfB1~>NlfYyd zM#;$sm_&I(LpT|jy%j=`B>muqfD{Tb`@;D(%o)txPzxUG2u&7diGj+AOg?L|EAvfIu1unfMtP($zsc084dGWB>pF diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index d8637604..429935d6 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -4,7 +4,8 @@ from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, BaseTest from tests.commons import ( TEST_MVN_BUCKET, COMMONS_CLIENT_456_FILES, COMMONS_CLIENT_METAS, - COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS + COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS, ARCHETYPE_CATALOG, + ARCHETYPE_CATALOG_FILES ) from moto import mock_s3 import boto3 @@ -48,9 +49,13 @@ def test_maven_deletion(self): for f in COMMONS_CLIENT_456_FILES: self.assertNotIn(f, actual_files) + for f in COMMONS_CLIENT_METAS: self.assertIn(f, actual_files) + for f in ARCHETYPE_CATALOG_FILES: + self.assertIn(f, actual_files) + for f in COMMONS_LOGGING_FILES: self.assertIn(f, actual_files) for f in COMMONS_LOGGING_METAS: @@ -74,6 +79,14 @@ def test_maven_deletion(self): self.assertIn("4.5.9", meta_content_client) self.assertIn("4.5.9", meta_content_client) + meta_obj_cat = test_bucket.Object(ARCHETYPE_CATALOG) + meta_content_cat = str(meta_obj_cat.get()["Body"].read(), "utf-8") + self.assertIn( + "org.apache.httpcomponents", meta_content_cat + ) + self.assertIn("httpclient", meta_content_cat) + self.assertNotIn("4.5.6", meta_content_cat) + meta_obj_logging = test_bucket.Object(COMMONS_LOGGING_METAS[0]) self.assertNotIn(PRODUCT_META_KEY, meta_obj_logging.metadata) meta_content_logging = str(meta_obj_logging.get()["Body"].read(), "utf-8") diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index 46b73cc9..f2e3c49f 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -6,7 +6,7 @@ from tests.commons import ( TEST_MVN_BUCKET, COMMONS_CLIENT_456_FILES, COMMONS_CLIENT_459_FILES, COMMONS_CLIENT_METAS, COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS, - NON_MVN_FILES + NON_MVN_FILES, ARCHETYPE_CATALOG, ARCHETYPE_CATALOG_FILES ) from moto import mock_s3 import boto3 @@ -44,17 +44,17 @@ def test_fresh_upload(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(18, len(actual_files)) + self.assertEqual(19, len(actual_files)) - for f in COMMONS_CLIENT_456_FILES: - self.assertIn(f, actual_files) - for f in COMMONS_CLIENT_METAS: - self.assertIn(f, actual_files) + filesets = [ + COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES, + COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS, + ARCHETYPE_CATALOG_FILES + ] - for f in COMMONS_LOGGING_FILES: - self.assertIn(f, actual_files) - for f in COMMONS_LOGGING_METAS: - self.assertIn(f, actual_files) + for fileset in filesets: + for f in fileset: + self.assertIn(f, actual_files) for f in NON_MVN_FILES: self.assertNotIn(f, actual_files) @@ -92,6 +92,12 @@ def test_fresh_upload(self): self.assertIn("1.2", meta_content_logging) self.assertIn("1.2", meta_content_logging) + catalog = test_bucket.Object(ARCHETYPE_CATALOG) + cat_content = str(catalog.get()["Body"].read(), "utf-8") + self.assertIn("4.5.6", cat_content) + self.assertIn("httpclient", cat_content) + self.assertIn("org.apache.httpcomponents", cat_content) + def test_overlap_upload(self): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" @@ -112,18 +118,15 @@ def test_overlap_upload(self): actual_files = [obj.key for obj in objs] self.assertEqual(22, len(actual_files)) - for f in COMMONS_CLIENT_456_FILES: - self.assertIn(f, actual_files) - self.assertEqual( - product_456, test_bucket.Object(f).metadata[PRODUCT_META_KEY] - ) - for f in COMMONS_CLIENT_459_FILES: - self.assertIn(f, actual_files) - self.assertEqual( - product_459, test_bucket.Object(f).metadata[PRODUCT_META_KEY] - ) - for f in COMMONS_CLIENT_METAS: - self.assertIn(f, actual_files) + filesets = [ + COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES, + COMMONS_CLIENT_459_FILES, + ARCHETYPE_CATALOG_FILES + ] + for fileset in filesets: + for f in fileset: + self.assertIn(f, actual_files) + product_mix = set([product_456, product_459]) for f in COMMONS_LOGGING_FILES: @@ -154,6 +157,13 @@ def test_overlap_upload(self): self.assertIn("1.2", meta_content_logging) self.assertIn("1.2", meta_content_logging) + catalog = test_bucket.Object(ARCHETYPE_CATALOG) + cat_content = str(catalog.get()["Body"].read(), "utf-8") + self.assertIn("4.5.6", cat_content) + self.assertIn("4.5.9", cat_content) + self.assertIn("httpclient", cat_content) + self.assertIn("org.apache.httpcomponents", cat_content) + def test_ignore_upload(self): test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" From c0f5213527e335f7ede0611c2aa7ea27c88d4adf Mon Sep 17 00:00:00 2001 From: Gang Li Date: Tue, 7 Dec 2021 17:11:54 +0800 Subject: [PATCH 2/3] Fix: archetype-catalog tests failure fix * archetype-catalog digest files generation in maven.py * some test failure fix --- charon/pkgs/maven.py | 135 +++++++++++++++++++++++-------------- charon/pkgs/npm.py | 3 +- charon/storage.py | 7 +- tests/test_maven_del.py | 6 +- tests/test_maven_index.py | 10 +-- tests/test_maven_upload.py | 11 +-- tests/test_pkgs_dryrun.py | 2 +- 7 files changed, 104 insertions(+), 70 deletions(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 063fd39b..3036bdf4 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -24,7 +24,7 @@ from charon.constants import (META_FILE_GEN_KEY, META_FILE_DEL_KEY, META_FILE_FAILED, MAVEN_METADATA_TEMPLATE, ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME) -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Tuple from jinja2 import Template from datetime import datetime from zipfile import ZipFile, BadZipFile @@ -220,18 +220,24 @@ def gen_meta_file(group_id, artifact_id: str, versions: list, root="/", digest=T except FileNotFoundError as e: raise e if digest: - md5_path = final_meta_path + ".md5" - sha1_path = final_meta_path + ".sha1" - sha256_path = final_meta_path + ".sha256" - if __gen_digest_file(md5_path, final_meta_path, HashType.MD5): - meta_files.append(md5_path) - if __gen_digest_file(sha1_path, final_meta_path, HashType.SHA1): - meta_files.append(sha1_path) - if __gen_digest_file(sha256_path, final_meta_path, HashType.SHA256): - meta_files.append(sha256_path) + meta_files.extend(__gen_all_digest_files(final_meta_path)) return meta_files +def __gen_all_digest_files(meta_file_path: str) -> List[str]: + md5_path = meta_file_path + ".md5" + sha1_path = meta_file_path + ".sha1" + sha256_path = meta_file_path + ".sha256" + digest_files = [] + if __gen_digest_file(md5_path, meta_file_path, HashType.MD5): + digest_files.append(md5_path) + if __gen_digest_file(sha1_path, meta_file_path, HashType.SHA1): + digest_files.append(sha1_path) + if __gen_digest_file(sha256_path, meta_file_path, HashType.SHA256): + digest_files.append(sha256_path) + return digest_files + + def __gen_digest_file(hash_file_path, meta_file_path: str, hashtype: HashType) -> bool: try: overwrite_file(hash_file_path, digest(meta_file_path, hashtype)) @@ -339,9 +345,11 @@ def handle_maven_uploading( # 8. Upload archetype-catalog.xml if it has changed if upload_archetype_file: + archetype_files = [os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)] + archetype_files.extend(__hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME)) logger.info("Start updating archetype-catalog.xml to s3") (_, _failed_metas) = s3_client.upload_metadatas( - meta_file_paths=[os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)], + meta_file_paths=archetype_files, bucket_name=bucket, product=None, root=top_level, @@ -365,8 +373,7 @@ def handle_maven_uploading( bucket_name=bucket, product=None, root=top_level, - key_prefix=prefix_, - digests=False + key_prefix=prefix_ ) failed_metas.extend(_failed_metas) logger.info("Index files updating done\n") @@ -481,10 +488,12 @@ def handle_maven_del( logger.info("archetype-catalog.xml files generation done\n") # 8. Upload or Delete archetype-catalog.xml if it has changed + archetype_files = [os.path.join(root, ARCHETYPE_CATALOG_FILENAME)] + archetype_files.extend(__hash_decorate_metadata(root, ARCHETYPE_CATALOG_FILENAME)) if archetype_action < 0: logger.info("Start updating archetype-catalog.xml to s3") (_, _failed_metas) = s3_client.delete_files( - file_paths=[os.path.join(root, ARCHETYPE_CATALOG_FILENAME)], + file_paths=archetype_files, bucket_name=bucket, product=None, root=top_level, @@ -493,7 +502,7 @@ def handle_maven_del( failed_metas.extend(_failed_metas) elif archetype_action > 0: (_, _failed_metas) = s3_client.upload_metadatas( - meta_file_paths=[os.path.join(root, ARCHETYPE_CATALOG_FILENAME)], + meta_file_paths=archetype_files, bucket_name=bucket, product=None, root=top_level, @@ -563,7 +572,8 @@ def _scan_paths(files_root: str, ignore_patterns: List[str], for name in names: path = os.path.join(root_dir, name) if top_level in root_dir: - # Let's wait to do the regex / pom examination until we know we're inside a valid root directory. + # Let's wait to do the regex / pom examination until we + # know we're inside a valid root directory. if _is_ignored(name, ignore_patterns): ignored_paths.append(name) continue @@ -607,10 +617,13 @@ def _generate_rollback_archetype_catalog( s3: S3Client, bucket: str, root: str, prefix: str = None ) -> int: - """Determine whether the local archive contains /archetype-catalog.xml in the repo contents. - If so, determine whether the archetype-catalog.xml is already available in the bucket. Merge (or unmerge) these - catalogs and return an integer, indicating whether the bucket file should be replaced (+1), deleted (-1), or, - in the case where no action is required, it will return NO-OP (0). + """Determine whether the local archive contains /archetype-catalog.xml + in the repo contents. + If so, determine whether the archetype-catalog.xml is already + available in the bucket. Merge (or unmerge) these catalogs and + return an integer, indicating whether the bucket file should be + replaced (+1), deleted (-1), or, in the case where no action is + required, it will return NO-OP (0). NOTE: There are three return values: - +1 - UPLOAD the local catalog with its rolled back changes @@ -623,8 +636,6 @@ def _generate_rollback_archetype_catalog( else: remote = ARCHETYPE_CATALOG_FILENAME - do_write = False - # If there is no local catalog, this is a NO-OP if os.path.exists(local): if not s3.file_exists_in_bucket(bucket, remote): @@ -632,13 +643,15 @@ def _generate_rollback_archetype_catalog( return 0 else: # If there IS a catalog in the bucket, we need to merge or un-merge it. - with open(local) as f: + with open(local, "rb") as f: local_archetypes = _parse_archetypes(f.read()) if len(local_archetypes) < 1: - # If there are no local archetypes in the catalog, there's nothing to do. + # If there are no local archetypes in the catalog, + # there's nothing to do. logger.warning( - "No archetypes found in local archetype-catalog.xml, even though the file exists! Skipping." + "No archetypes found in local archetype-catalog.xml, " + "even though the file exists! Skipping." ) return 0 @@ -649,26 +662,34 @@ def _generate_rollback_archetype_catalog( if len(remote_archetypes) < 1: # Nothing in the bucket. Clear out this empty file. + __gen_all_digest_files(local) return -1 else: - # If we're deleting, un-merge the local archetypes from the remote ones. + # If we're deleting, un-merge the local archetypes from + # the remote ones. # - # NOTE: The ONLY reason we can get away with this kind of naive un-merge is that products only - # bother to publish archetypes for their own direct users. If they publish an archetype, it's only - # for use with their product. Therefore, if we rollback that product, the archetypes they reference - # shouldn't be useful anymore. + # NOTE: The ONLY reason we can get away with this kind of + # naive un-merge is that products only bother to publish + # archetypes for their own direct users. If they publish + # an archetype, it's only for use with their product. + # Therefore, if we rollback that product, the archetypes + # they reference shouldn't be useful anymore. for la in local_archetypes: if la not in remote_archetypes: remote_archetypes.remove(la) if len(remote_archetypes) < 1: - # If there are no remote archetypes left after removing ours DELETE the bucket catalog. + # If there are no remote archetypes left after removing + # ours DELETE the bucket catalog. + __gen_all_digest_files(local) return -1 else: - # Re-render the result of our archetype un-merge to the local file, in preparation for upload. - with open(local, 'w') as f: - content = MavenArchetypeCatalog(remote_archetypes).generate_meta_file_content() + # Re-render the result of our archetype un-merge to the + # local file, in preparation for upload. + with open(local, 'wb') as f: + content = MavenArchetypeCatalog(remote_archetypes)\ + .generate_meta_file_content() try: overwrite_file(local, content) except FileNotFoundError as e: @@ -677,7 +698,7 @@ def _generate_rollback_archetype_catalog( local, ) raise e - + __gen_all_digest_files(local) return 1 return 0 @@ -687,9 +708,11 @@ def _generate_upload_archetype_catalog( s3: S3Client, bucket: str, root: str, prefix: str = None ) -> bool: - """Determine whether the local archive contains /archetype-catalog.xml in the repo contents. - If so, determine whether the archetype-catalog.xml is already available in the bucket. Merge (or unmerge) these - catalogs and return a boolean indicating whether the local file should be uploaded. + """Determine whether the local archive contains /archetype-catalog.xml + in the repo contents. + If so, determine whether the archetype-catalog.xml is already + available in the bucket. Merge (or unmerge) these catalogs and + return a boolean indicating whether the local file should be uploaded. """ local = os.path.join(root, ARCHETYPE_CATALOG_FILENAME) if prefix: @@ -700,16 +723,18 @@ def _generate_upload_archetype_catalog( # If there is no local catalog, this is a NO-OP if os.path.exists(local): if not s3.file_exists_in_bucket(bucket, remote): + __gen_all_digest_files(local) # If there is no catalog in the bucket, just upload what we have locally return True else: # If there IS a catalog in the bucket, we need to merge or un-merge it. - with open(local) as f: + with open(local, "rb") as f: local_archetypes = _parse_archetypes(f.read()) if len(local_archetypes) < 1: logger.warning( - "No archetypes found in local archetype-catalog.xml, even though the file exists! Skipping." + "No archetypes found in local archetype-catalog.xml, " + "even though the file exists! Skipping." ) else: @@ -718,14 +743,17 @@ def _generate_upload_archetype_catalog( remote_archetypes = _parse_archetypes(remote_xml) if len(remote_archetypes) < 1: + __gen_all_digest_files(local) # Nothing in the bucket. Just push what we have locally. return True else: original_remote_size = len(remote_archetypes) for la in local_archetypes: - # The cautious approach in this operation contradicts assumptions we make for the rollback case. - # That's because we should NEVER encounter a collision on archetype GAV...they should - # belong with specific product releases. + # The cautious approach in this operation contradicts + # assumptions we make for the rollback case. + # That's because we should NEVER encounter a collision + # on archetype GAV...they should belong with specific + # product releases. # # Still, we will WARN, not ERROR if we encounter this. if la not in remote_archetypes: @@ -738,13 +766,16 @@ def _generate_upload_archetype_catalog( ) if len(remote_archetypes) != original_remote_size: - # If the number of archetypes in the version of the file from the bucket has changed, we need + # If the number of archetypes in the version of + # the file from the bucket has changed, we need # to regenerate the file and re-upload it. # - # Re-render the result of our archetype merge / un-merge to the local file, in preparation for + # Re-render the result of our archetype merge / + # un-merge to the local file, in preparation for # upload. - with open(local, 'w') as f: - content = MavenArchetypeCatalog(remote_archetypes).generate_meta_file_content() + with open(local, 'wb') as f: + content = MavenArchetypeCatalog(remote_archetypes)\ + .generate_meta_file_content() try: overwrite_file(local, content) except FileNotFoundError as e: @@ -753,14 +784,17 @@ def _generate_upload_archetype_catalog( local, ) raise e - + __gen_all_digest_files(local) return True return False def _parse_archetypes(source) -> List[ArchetypeRef]: - tree = ElementTree.fromstring(source.strip(), forbid_dtd=True, forbid_entities=True, forbid_external=True) + tree = ElementTree.fromstring( + source.strip(), forbid_dtd=True, + forbid_entities=True, forbid_external=True + ) archetypes = [] for a in tree.findall("./archetypes/archetype"): gid = a.find('groupId').text @@ -944,6 +978,7 @@ def __init__(self, gav): super().__init__(gav.version) self.gav = gav + # pylint: disable=unused-private-member def __compare(self, other) -> int: x = self.gav.group_id + ":" + self.gav.artifact_id y = other.gav.group_id + ":" + other.gav.artifact_id @@ -954,5 +989,3 @@ def __compare(self, other) -> int: return -1 else: return 1 - - diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index c4a7feb5..5b78a5e7 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -131,8 +131,7 @@ def handle_npm_uploading( (_, _failed_metas) = client.upload_metadatas( meta_file_paths=created_indexes, bucket_name=bucket, product=None, - root=target_dir, key_prefix=prefix_, - digests=False + root=target_dir, key_prefix=prefix_ ) failed_metas.extend(_failed_metas) logger.info("Index files updating done\n") diff --git a/charon/storage.py b/charon/storage.py index 17fa13ac..51d02dc9 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -288,9 +288,10 @@ def path_delete_handler(full_file_path: str, path: str): fileObject = bucket.Object(path_key) existed = self.file_exists(fileObject) if existed: - # NOTE: If we're NOT using the product key to track collisions (in the case of metadata), then - # This prods array will remain empty, and we will just delete the file, below. Otherwise, the product - # reference counts will be used (from object metadata). + # NOTE: If we're NOT using the product key to track collisions + # (in the case of metadata), then this prods array will remain + # empty, and we will just delete the file, below. Otherwise, + # the product reference counts will be used (from object metadata). prods = [] if product: try: diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index 429935d6..688b2018 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -45,7 +45,7 @@ def test_maven_deletion(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(18, len(actual_files)) + self.assertEqual(22, len(actual_files)) for f in COMMONS_CLIENT_456_FILES: self.assertNotIn(f, actual_files) @@ -125,7 +125,7 @@ def test_ignore_del(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(20, len(actual_files)) + self.assertEqual(24, len(actual_files)) httpclient_ignored_files = [ "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom.sha1", @@ -186,7 +186,7 @@ def __test_prefix_deletion(self, prefix: str): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(18, len(actual_files)) + self.assertEqual(22, len(actual_files)) prefix_ = remove_prefix(prefix, "/") PREFIXED_COMMONS_CLIENT_456_FILES = [ diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py index 2ed718b7..9df90efd 100644 --- a/tests/test_maven_index.py +++ b/tests/test_maven_index.py @@ -60,7 +60,7 @@ def test_uploading_index(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(27, len(actual_files)) + self.assertEqual(31, len(actual_files)) for f in COMMONS_LOGGING_INDEXES: self.assertIn(f, actual_files) @@ -113,7 +113,7 @@ def test_overlap_upload_index(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) - self.assertEqual(32, len(objs)) + self.assertEqual(36, len(objs)) indedx_obj = test_bucket.Object(COMMONS_CLIENT_INDEX) index_content = str(indedx_obj.get()["Body"].read(), "utf-8") @@ -164,7 +164,7 @@ def __test_upload_index_with_prefix(self, prefix: str): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(27, len(actual_files)) + self.assertEqual(31, len(actual_files)) prefix_ = remove_prefix(prefix, "/") PREFIXED_LOGGING_INDEXES = [ @@ -221,7 +221,7 @@ def test_deletion_index(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(27, len(actual_files)) + self.assertEqual(31, len(actual_files)) for assert_file in COMMONS_CLIENT_459_INDEXES: self.assertIn(assert_file, actual_files) @@ -288,7 +288,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(27, len(actual_files)) + self.assertEqual(31, len(actual_files)) prefix_ = remove_prefix(prefix, "/") PREFIXED_459_INDEXES = [os.path.join(prefix_, i) for i in COMMONS_CLIENT_459_INDEXES] diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index f2e3c49f..a2549a8f 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -38,13 +38,14 @@ def test_fresh_upload(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, do_index=False + bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, + do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(19, len(actual_files)) + self.assertEqual(22, len(actual_files)) filesets = [ COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES, @@ -116,7 +117,7 @@ def test_overlap_upload(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(22, len(actual_files)) + self.assertEqual(26, len(actual_files)) filesets = [ COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES, @@ -175,7 +176,7 @@ def test_ignore_upload(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(13, len(actual_files)) + self.assertEqual(17, len(actual_files)) ignored_files = [ "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom.sha1", @@ -211,7 +212,7 @@ def __test_prefix_upload(self, prefix: str): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] - self.assertEqual(18, len(actual_files)) + self.assertEqual(22, len(actual_files)) prefix_ = remove_prefix(prefix, "/") PREFIXED_COMMONS_CLIENT_456_FILES = [ diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py index f936c6b8..971822f5 100644 --- a/tests/test_pkgs_dryrun.py +++ b/tests/test_pkgs_dryrun.py @@ -55,7 +55,7 @@ def test_maven_delete_dry_run(self): test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) objs = list(test_bucket.objects.all()) - self.assertEqual(32, len(objs)) + self.assertEqual(36, len(objs)) def test_npm_upload_dry_run(self): test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz") From 6ddf0585eeb49c6b1350702c8fbf33bc287c87d9 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Tue, 7 Dec 2021 18:59:07 +0800 Subject: [PATCH 3/3] Fix: some failure tests fix maven_del tests and maven_index tests --- charon/pkgs/maven.py | 8 ++++---- tests/test_maven_del.py | 16 +++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 3036bdf4..3790132f 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -126,7 +126,7 @@ def __init__(self, archetypes: List[ArchetypeRef]): def generate_meta_file_content(self) -> str: template = Template(ARCHETYPE_CATALOG_TEMPLATE) - return template.render(meta=self) + return template.render(archetypes=self.archetypes) def __str__(self) -> str: return f"(Archetype Catalog with {len(self.archetypes)} entries).\n\n" @@ -488,8 +488,8 @@ def handle_maven_del( logger.info("archetype-catalog.xml files generation done\n") # 8. Upload or Delete archetype-catalog.xml if it has changed - archetype_files = [os.path.join(root, ARCHETYPE_CATALOG_FILENAME)] - archetype_files.extend(__hash_decorate_metadata(root, ARCHETYPE_CATALOG_FILENAME)) + archetype_files = [os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)] + archetype_files.extend(__hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME)) if archetype_action < 0: logger.info("Start updating archetype-catalog.xml to s3") (_, _failed_metas) = s3_client.delete_files( @@ -676,7 +676,7 @@ def _generate_rollback_archetype_catalog( # Therefore, if we rollback that product, the archetypes # they reference shouldn't be useful anymore. for la in local_archetypes: - if la not in remote_archetypes: + if la in remote_archetypes: remote_archetypes.remove(la) if len(remote_archetypes) < 1: diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index 688b2018..2aeeb27e 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -39,7 +39,8 @@ def test_maven_deletion(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, do_index=False + bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, + do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET) @@ -50,15 +51,12 @@ def test_maven_deletion(self): for f in COMMONS_CLIENT_456_FILES: self.assertNotIn(f, actual_files) - for f in COMMONS_CLIENT_METAS: - self.assertIn(f, actual_files) - - for f in ARCHETYPE_CATALOG_FILES: - self.assertIn(f, actual_files) + file_set = [ + *COMMONS_CLIENT_METAS, *ARCHETYPE_CATALOG_FILES, + *COMMONS_LOGGING_FILES, *COMMONS_LOGGING_METAS + ] - for f in COMMONS_LOGGING_FILES: - self.assertIn(f, actual_files) - for f in COMMONS_LOGGING_METAS: + for f in file_set: self.assertIn(f, actual_files) for obj in objs: