From adda6ac56097ea396997f2f50ea0ecf32e540744 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Mon, 27 Feb 2023 16:42:58 +0200 Subject: [PATCH 1/7] managing md5, wrapping pullipinfo Signed-off-by: yoavkatzman --- .../client/curieconf/cli/__init__.py | 36 ++++++++++++------- curiefense/images/curiesync/init/pull.sh | 11 ++++-- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/curiefense/curieconf/client/curieconf/cli/__init__.py b/curiefense/curieconf/client/curieconf/cli/__init__.py index 420365891..500bd2e94 100755 --- a/curiefense/curieconf/client/curieconf/cli/__init__.py +++ b/curiefense/curieconf/client/curieconf/cli/__init__.py @@ -496,26 +496,35 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): os.makedirs(target_path, exist_ok=True) for ipinfo_blob in [*(client.list_blobs(bucket_or_name=bucket, prefix=ipinfo_dir))]: + remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() file_name = ipinfo_blob.name.split("/")[-1] if not os.path.isfile(target_path + file_name): try: ipinfo_blob.download_to_filename(target_path + file_name) + with open(target_path+file_name.split(".")[0]+"_hash", "w") as hash_file: + hash_file.write(remote_md5) except Exception as ex: - logger.error(f"failed downloading {file_name} - {ex}") + typer.echo(f"failed downloading {file_name} - {ex}", ) continue - logger.info(f"downloaded {file_name} to {target_path}") + typer.echo(f"downloaded {file_name} to {target_path}") else: - remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() - local_md5 = get_md5(target_path + file_name) - if not local_md5 == remote_md5: - try: - ipinfo_blob.download_to_filename(target_path + file_name) - except Exception as ex: - logger.error(f"failed downloading {file_name} - {ex}") - continue + with open(target_path+file_name.split(".")[0]+"_hash", "r+") as hash_file: + + remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() + local_md5 = hash_file.readline().rstrip() + if not local_md5 == remote_md5: + try: + ipinfo_blob.download_to_filename(target_path + file_name) + hash_file.seek(0) + hash_file.write(remote_md5) + hash_file.truncate() + except Exception as ex: + logger.error(f"failed downloading {file_name} - {ex}") + continue + + else: + typer.echo(f"{target_path} already contains the updated {file_name}") - else: - logger.info(f"{target_path} already contains the updated {file_name}") @sync.command() @@ -655,4 +664,5 @@ def main(): if __name__ == "__main__": - main() + #main() + pullipinfo("rbz-internal", "rbz-dev-auto-acl", "ipinfo", "/tmp/ipinfo/") diff --git a/curiefense/images/curiesync/init/pull.sh b/curiefense/images/curiesync/init/pull.sh index 58e100b83..27b90f786 100755 --- a/curiefense/images/curiesync/init/pull.sh +++ b/curiefense/images/curiesync/init/pull.sh @@ -20,8 +20,10 @@ fi if [ "$RUN_MODE" = "SYNC_ONCE" ]; then info "Synchronizing once" curieconfctl sync pull "${CURIE_BUCKET_LINK}" /cf-config - curieconfctl sync pullipinfo rbz-internal rbz-dev-auto-acl ipinfo /cf-config/ipinfo/ - + if [ "$USE_IPINFO" = "true" ] + then + curieconfctl sync pullipinfo rbz-internal rbz-dev-auto-acl ipinfo /cf-config/ipinfo/ + fi exit 0 fi @@ -49,8 +51,11 @@ if [ "$RUN_MODE" = "PERIODIC_SYNC" ] || [ -z "$RUN_MODE" ]; then while :; do info "Pulling ${CURIE_BUCKET_LINK}" - curieconfctl sync pullipinfo rbz-internal rbz-dev-auto-acl ipinfo /cf-config/ipinfo/ curieconfctl sync pull "${CURIE_BUCKET_LINK}" /cf-config --on-conf-change "$CONFIGCHANGE" + if [ "$USE_IPINFO" = "true" ] + then + curieconfctl sync pullipinfo rbz-internal rbz-dev-auto-acl ipinfo /cf-config/ipinfo/ + fi info "Sleeping" sleep $PERIOD done From c1e99f72d1a82cec20392541df6dd557b7bfd6ab Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Tue, 28 Feb 2023 16:09:26 +0200 Subject: [PATCH 2/7] env variables Signed-off-by: yoavkatzman --- curiefense/curieconf/client/curieconf/cli/__init__.py | 4 ++-- curiefense/images/curiesync/init/pull.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/curiefense/curieconf/client/curieconf/cli/__init__.py b/curiefense/curieconf/client/curieconf/cli/__init__.py index 500bd2e94..897c271f7 100755 --- a/curiefense/curieconf/client/curieconf/cli/__init__.py +++ b/curiefense/curieconf/client/curieconf/cli/__init__.py @@ -495,6 +495,7 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): os.makedirs(target_path, exist_ok=True) + for ipinfo_blob in [*(client.list_blobs(bucket_or_name=bucket, prefix=ipinfo_dir))]: remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() file_name = ipinfo_blob.name.split("/")[-1] @@ -664,5 +665,4 @@ def main(): if __name__ == "__main__": - #main() - pullipinfo("rbz-internal", "rbz-dev-auto-acl", "ipinfo", "/tmp/ipinfo/") + main() diff --git a/curiefense/images/curiesync/init/pull.sh b/curiefense/images/curiesync/init/pull.sh index 27b90f786..5314cd094 100755 --- a/curiefense/images/curiesync/init/pull.sh +++ b/curiefense/images/curiesync/init/pull.sh @@ -22,7 +22,7 @@ if [ "$RUN_MODE" = "SYNC_ONCE" ]; then curieconfctl sync pull "${CURIE_BUCKET_LINK}" /cf-config if [ "$USE_IPINFO" = "true" ] then - curieconfctl sync pullipinfo rbz-internal rbz-dev-auto-acl ipinfo /cf-config/ipinfo/ + curieconfctl sync pullipinfo "${IPINFO_PROJECT}" "${IPINFO_REMOTE_BUCKET}" "${IPINFO_REMOTE_DIR}" /cf-config/ipinfo/ fi exit 0 fi @@ -54,7 +54,7 @@ if [ "$RUN_MODE" = "PERIODIC_SYNC" ] || [ -z "$RUN_MODE" ]; then curieconfctl sync pull "${CURIE_BUCKET_LINK}" /cf-config --on-conf-change "$CONFIGCHANGE" if [ "$USE_IPINFO" = "true" ] then - curieconfctl sync pullipinfo rbz-internal rbz-dev-auto-acl ipinfo /cf-config/ipinfo/ + curieconfctl sync pullipinfo "${IPINFO_PROJECT}" "${IPINFO_REMOTE_BUCKET}" "${IPINFO_REMOTE_DIR}" /cf-config/ipinfo/ fi info "Sleeping" sleep $PERIOD From 88abebc5256b5fcec13dfac7337efd7c0a7c6206 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Tue, 28 Feb 2023 18:08:53 +0200 Subject: [PATCH 3/7] black Signed-off-by: Curiefense Bootstrap Script --- .../curieconf/client/curieconf/cli/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/curiefense/curieconf/client/curieconf/cli/__init__.py b/curiefense/curieconf/client/curieconf/cli/__init__.py index 897c271f7..3a6eae2fb 100755 --- a/curiefense/curieconf/client/curieconf/cli/__init__.py +++ b/curiefense/curieconf/client/curieconf/cli/__init__.py @@ -495,21 +495,26 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): os.makedirs(target_path, exist_ok=True) - for ipinfo_blob in [*(client.list_blobs(bucket_or_name=bucket, prefix=ipinfo_dir))]: remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() file_name = ipinfo_blob.name.split("/")[-1] if not os.path.isfile(target_path + file_name): try: ipinfo_blob.download_to_filename(target_path + file_name) - with open(target_path+file_name.split(".")[0]+"_hash", "w") as hash_file: + with open( + target_path + file_name.split(".")[0] + "_hash", "w" + ) as hash_file: hash_file.write(remote_md5) except Exception as ex: - typer.echo(f"failed downloading {file_name} - {ex}", ) + typer.echo( + f"failed downloading {file_name} - {ex}", + ) continue typer.echo(f"downloaded {file_name} to {target_path}") else: - with open(target_path+file_name.split(".")[0]+"_hash", "r+") as hash_file: + with open( + target_path + file_name.split(".")[0] + "_hash", "r+" + ) as hash_file: remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() local_md5 = hash_file.readline().rstrip() @@ -524,8 +529,9 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): continue else: - typer.echo(f"{target_path} already contains the updated {file_name}") - + typer.echo( + f"{target_path} already contains the updated {file_name}" + ) @sync.command() From 8b8bb5b529e7ab2076e0c891d688de7c16d239c2 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Tue, 28 Feb 2023 18:10:51 +0200 Subject: [PATCH 4/7] black Signed-off-by: Curiefense Bootstrap Script --- curiefense/curieconf/client/curieconf/cli/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/curiefense/curieconf/client/curieconf/cli/__init__.py b/curiefense/curieconf/client/curieconf/cli/__init__.py index 3a6eae2fb..066bc0568 100755 --- a/curiefense/curieconf/client/curieconf/cli/__init__.py +++ b/curiefense/curieconf/client/curieconf/cli/__init__.py @@ -515,7 +515,6 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): with open( target_path + file_name.split(".")[0] + "_hash", "r+" ) as hash_file: - remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() local_md5 = hash_file.readline().rstrip() if not local_md5 == remote_md5: From 97b4ce89d32b947f2c4602912d8091db4092cb3c Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 2 Mar 2023 16:56:40 +0200 Subject: [PATCH 5/7] PR fixes Signed-off-by: Curiefense Bootstrap Script --- .../client/curieconf/cli/__init__.py | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/curiefense/curieconf/client/curieconf/cli/__init__.py b/curiefense/curieconf/client/curieconf/cli/__init__.py index 066bc0568..af350563b 100755 --- a/curiefense/curieconf/client/curieconf/cli/__init__.py +++ b/curiefense/curieconf/client/curieconf/cli/__init__.py @@ -498,39 +498,34 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): for ipinfo_blob in [*(client.list_blobs(bucket_or_name=bucket, prefix=ipinfo_dir))]: remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() file_name = ipinfo_blob.name.split("/")[-1] + hash_file_name = target_path + file_name.split(".")[0] + "_hash" if not os.path.isfile(target_path + file_name): try: ipinfo_blob.download_to_filename(target_path + file_name) with open( - target_path + file_name.split(".")[0] + "_hash", "w" + hash_file_name, "w" ) as hash_file: hash_file.write(remote_md5) except Exception as ex: - typer.echo( - f"failed downloading {file_name} - {ex}", - ) + typer.echo(f"failed downloading {file_name} - {ex}", err=True) continue typer.echo(f"downloaded {file_name} to {target_path}") else: - with open( - target_path + file_name.split(".")[0] + "_hash", "r+" - ) as hash_file: - remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() + with open(hash_file_name, "r") as hash_file: local_md5 = hash_file.readline().rstrip() - if not local_md5 == remote_md5: + + if not local_md5 == remote_md5: + with open(hash_file_name, "w") as hash_file: try: ipinfo_blob.download_to_filename(target_path + file_name) - hash_file.seek(0) hash_file.write(remote_md5) - hash_file.truncate() + typer.echo(f"updated {file_name} in {target_path}") except Exception as ex: - logger.error(f"failed downloading {file_name} - {ex}") + typer.echo(f"failed downloading {file_name} - {ex}", err=True) continue - else: - typer.echo( - f"{target_path} already contains the updated {file_name}" - ) + else: + typer.echo(f"{target_path} already contains the updated {file_name}") @sync.command() From 17b59c9efe694802e4ef5a7088fe1bbb2d627c49 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 2 Mar 2023 17:05:53 +0200 Subject: [PATCH 6/7] black Signed-off-by: Curiefense Bootstrap Script --- curiefense/curieconf/client/curieconf/cli/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/curiefense/curieconf/client/curieconf/cli/__init__.py b/curiefense/curieconf/client/curieconf/cli/__init__.py index af350563b..3fa1a5370 100755 --- a/curiefense/curieconf/client/curieconf/cli/__init__.py +++ b/curiefense/curieconf/client/curieconf/cli/__init__.py @@ -502,9 +502,7 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): if not os.path.isfile(target_path + file_name): try: ipinfo_blob.download_to_filename(target_path + file_name) - with open( - hash_file_name, "w" - ) as hash_file: + with open(hash_file_name, "w") as hash_file: hash_file.write(remote_md5) except Exception as ex: typer.echo(f"failed downloading {file_name} - {ex}", err=True) From 25ceadb0df5a77e058eb71e26b5384d13c700106 Mon Sep 17 00:00:00 2001 From: Evgeniy Leybovich Date: Sun, 5 Mar 2023 10:48:10 +0200 Subject: [PATCH 7/7] use pathlib.Path for parsing --- curiefense/curieconf/client/curieconf/cli/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/curiefense/curieconf/client/curieconf/cli/__init__.py b/curiefense/curieconf/client/curieconf/cli/__init__.py index 3fa1a5370..e7fa822f2 100755 --- a/curiefense/curieconf/client/curieconf/cli/__init__.py +++ b/curiefense/curieconf/client/curieconf/cli/__init__.py @@ -9,6 +9,7 @@ import json import subprocess from enum import Enum +from pathlib import Path from google.cloud import storage import base64 from typing import List, Optional @@ -497,8 +498,9 @@ def pullipinfo(project: str, bucket: str, ipinfo_dir: str, target_path: str): for ipinfo_blob in [*(client.list_blobs(bucket_or_name=bucket, prefix=ipinfo_dir))]: remote_md5 = base64.b64decode(ipinfo_blob.md5_hash).hex() - file_name = ipinfo_blob.name.split("/")[-1] - hash_file_name = target_path + file_name.split(".")[0] + "_hash" + path = Path(ipinfo_blob.name) + file_name = path.name + hash_file_name = target_path + path.stem + "_hash" if not os.path.isfile(target_path + file_name): try: ipinfo_blob.download_to_filename(target_path + file_name)