From a6ebcbb0f002befec487c57cfb758e3a82e834a5 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 28 Mar 2024 14:48:14 +0800 Subject: [PATCH 1/2] Change cf cmd name to cf-invalidate --- charon/cmd/__init__.py | 4 ++-- charon/cmd/cmd_cache.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index 7a22f570..9f62b768 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -18,7 +18,7 @@ from charon.cmd.cmd_delete import delete from charon.cmd.cmd_index import index from charon.cmd.cmd_checksum import validate -from charon.cmd.cmd_cache import clear_cf +from charon.cmd.cmd_cache import cf_invalidate @group() @@ -34,4 +34,4 @@ def cli(): cli.add_command(delete) cli.add_command(index) cli.add_command(validate) -cli.add_command(clear_cf) +cli.add_command(cf_invalidate) diff --git a/charon/cmd/cmd_cache.py b/charon/cmd/cmd_cache.py index e6cd61a4..348b1cb0 100644 --- a/charon/cmd/cmd_cache.py +++ b/charon/cmd/cmd_cache.py @@ -76,7 +76,7 @@ default=False ) @command() -def clear_cf( +def cf_invalidate( target: str, paths: List[str], path_file: str, From 626a346a1b27b4aa3c2264a83d01f9601da47f59 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 28 Mar 2024 15:25:16 +0800 Subject: [PATCH 2/2] Add command to do CF invalidation status check --- charon/cmd/__init__.py | 3 +- charon/cmd/cmd_cache.py | 118 +++++++++++++++++++++++++++++++++------ charon/cmd/internal.py | 4 +- charon/pkgs/pkg_utils.py | 7 ++- 4 files changed, 112 insertions(+), 20 deletions(-) diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index 9f62b768..a2ccadaa 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -18,7 +18,7 @@ from charon.cmd.cmd_delete import delete from charon.cmd.cmd_index import index from charon.cmd.cmd_checksum import validate -from charon.cmd.cmd_cache import cf_invalidate +from charon.cmd.cmd_cache import cf_invalidate, cf_check @group() @@ -35,3 +35,4 @@ def cli(): cli.add_command(index) cli.add_command(validate) cli.add_command(cf_invalidate) +cli.add_command(cf_check) diff --git a/charon/cmd/cmd_cache.py b/charon/cmd/cmd_cache.py index 348b1cb0..ed874723 100644 --- a/charon/cmd/cmd_cache.py +++ b/charon/cmd/cmd_cache.py @@ -18,8 +18,8 @@ from charon.cmd.internal import _decide_mode, _get_buckets from charon.cache import CFClient from charon.pkgs.pkg_utils import invalidate_cf_paths -from click import command, option -from typing import List +from click import command, option, argument +from typing import List, Tuple import traceback import logging @@ -34,9 +34,8 @@ "-t", "target", help=""" - The target to do the uploading, which will decide which s3 bucket - and what root path where all files will be uploaded to. - Can accept more than one target. + The target to do the invalidating, which will decide the s3 bucket + which and its related domain to get the distribution. """, required=True ) @@ -104,23 +103,110 @@ def cf_invalidate( for line in f.readlines(): work_paths.append(str(line).strip()) + use_wildcard = False + for path in work_paths: + if "*" in path: + use_wildcard = True + break + try: - conf = get_config() - if not conf: - sys.exit(1) + (buckets, aws_profile) = _init_cmd(target) - aws_profile = os.getenv("AWS_PROFILE") or conf.get_aws_profile() - if not aws_profile: - logger.error("No AWS profile specified!") - sys.exit(1) + for b in buckets: + cf_client = CFClient(aws_profile=aws_profile) + # Per aws official doc, if the paths contains wildcard, it is + # limited to 15 as max items in one request. Otherwise it could + # be 3000 + if use_wildcard: + invalidate_cf_paths( + cf_client, b, work_paths + ) + else: + invalidate_cf_paths( + cf_client, b, work_paths, b, batch_size=3000 + ) + except Exception: + print(traceback.format_exc()) + sys.exit(2) - buckets = _get_buckets([target], conf) +@argument( + "invalidation_id", + type=str +) +@option( + "--target", + "-t", + "target", + help=""" + The target to do the invalidating, which will decide the s3 bucket + which and its related domain to get the distribution. + """, + required=True +) +@option( + "--debug", + "-D", + "debug", + help="Debug mode, will print all debug logs for problem tracking.", + is_flag=True, + default=False +) +@option( + "--quiet", + "-q", + "quiet", + help="Quiet mode, will shrink most of the logs except warning and errors.", + is_flag=True, + default=False +) +@command() +def cf_check( + invalidation_id: str, + target: str, + quiet: bool = False, + debug: bool = False +): + """This command will check the invalidation status of the specified invalidation id. + """ + _decide_mode( + f"cfcheck-{target}", "", + is_quiet=quiet, is_debug=debug + ) + try: + (buckets, aws_profile) = _init_cmd(target) for b in buckets: cf_client = CFClient(aws_profile=aws_profile) - invalidate_cf_paths( - cf_client, b, work_paths - ) + bucket_name = b[1] + domain = b[4] + if not domain: + domain = cf_client.get_domain_by_bucket(bucket_name) + if domain: + distr_id = cf_client.get_dist_id_by_domain(domain) + if distr_id: + result = cf_client.check_invalidation(distr_id, invalidation_id) + logger.info( + "The status of invalidation %s is %s", + invalidation_id, result + ) + else: + logger.error( + "Can not check invalidation result for %s because domain not found" + " for bucket %s. ", invalidation_id, bucket_name + ) except Exception: print(traceback.format_exc()) sys.exit(2) + + +def _init_cmd(target: str) -> Tuple[List[Tuple[str, str, str, str, str]], str]: + conf = get_config() + if not conf: + sys.exit(1) + + aws_profile = os.getenv("AWS_PROFILE") or conf.get_aws_profile() + if not aws_profile: + logger.error("No AWS profile specified!") + sys.exit(1) + + return (_get_buckets([target], conf), aws_profile) diff --git a/charon/cmd/internal.py b/charon/cmd/internal.py index edc87c05..e901c8f4 100644 --- a/charon/cmd/internal.py +++ b/charon/cmd/internal.py @@ -28,7 +28,9 @@ logger = logging.getLogger(__name__) -def _get_buckets(targets: List[str], conf: CharonConfig) -> List[Tuple[str, str, str, str]]: +def _get_buckets( + targets: List[str], conf: CharonConfig +) -> List[Tuple[str, str, str, str, str]]: buckets = [] for target in targets: for bucket in conf.get_target(target): diff --git a/charon/pkgs/pkg_utils.py b/charon/pkgs/pkg_utils.py index 00fa293a..ce0d20ff 100644 --- a/charon/pkgs/pkg_utils.py +++ b/charon/pkgs/pkg_utils.py @@ -66,7 +66,8 @@ def invalidate_cf_paths( cf_client: CFClient, bucket: Tuple[str, str, str, str, str], invalidate_paths: List[str], - root="/" + root="/", + batch_size=15 ): logger.info("Invalidating CF cache for %s", bucket[1]) bucket_name = bucket[1] @@ -90,7 +91,9 @@ def invalidate_cf_paths( if domain: distr_id = cf_client.get_dist_id_by_domain(domain) if distr_id: - result = cf_client.invalidate_paths(distr_id, final_paths) + result = cf_client.invalidate_paths( + distr_id, final_paths, batch_size + ) if result: logger.info( "The CF invalidating request for metadata/indexing is sent, "