From 46f4cda2064fd69d03ceffcef8dd6a5322b7d26d Mon Sep 17 00:00:00 2001 From: Pete Stenger Date: Tue, 15 Nov 2022 16:28:31 -0600 Subject: [PATCH 1/9] Add draft code for verify and code CLI options --- README.md | 19 ++++++ ctfcli/cli/challenges.py | 46 ++++++++++++++ ctfcli/utils/challenge.py | 130 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+) diff --git a/README.md b/README.md index a3e08d3..44027ff 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,25 @@ Syncing buffer_overflow Success! ``` +## 5. Verify & pull challenges + +If, at any point, changes may have been made to a challenge through the CTFd UI by an admin. To verify that your challenge.yml file matches remote contents, you can use: + +``` +❯ ctf challenge verify [challenge.yml | DIRECTORY] [--verify-contents] +``` + +If the `--verify-contents` flag is set, challenge files will be downloaded and the binary contents will be compared. + +If you want to pull down the latest version of the challenge, and its challenge files, you can use: + +``` +❯ ctf challenge verify [challenge.yml | DIRECTORY] [--create-files] +``` + +If the `--create-files` flag is set, any new files added to through the CTFd UI will be downloaded to the same directory as the `challenge.yml` file. + +**This is a destructive action! It will overwrite the local version of `challenge.yml` with the version on CTFd!** # Challenge Templates `ctfcli` contains pre-made challenge templates to make it faster to create CTF challenges with safe defaults. diff --git a/ctfcli/cli/challenges.py b/ctfcli/cli/challenges.py index fd960cb..5d1a2b6 100644 --- a/ctfcli/cli/challenges.py +++ b/ctfcli/cli/challenges.py @@ -15,6 +15,8 @@ load_installed_challenge, load_installed_challenges, sync_challenge, + verify_challenge, + pull_challenge ) from ctfcli.utils.config import ( get_base_path, @@ -414,3 +416,47 @@ def healthcheck(self, challenge): f"Success", fg="green", ) sys.exit(0) + + def verify(self, challenge=None, verify_contents=False): + if challenge is None: + # Get all challenges if not specifying a challenge + config = load_config() + challenges = dict(config["challenges"]).keys() + else: + challenges = [challenge] + + for challenge in challenges: + path = Path(challenge) + + if path.name.endswith(".yml") is False: + path = path / "challenge.yml" + + click.secho(f"Found {path}") + challenge = load_challenge(path) + click.secho(f'Loaded {challenge["name"]}', fg="yellow") + + click.secho(f'Verifying {challenge["name"]}', fg="yellow") + verify_challenge(challenge=challenge, verify_contents=verify_contents) + click.secho(f"Success!", fg="green") + + def pull(self, challenge=None, create_files=False): + if challenge is None: + # Get all challenges if not specifying a challenge + config = load_config() + challenges = dict(config["challenges"]).keys() + else: + challenges = [challenge] + + for challenge in challenges: + path = Path(challenge) + + if path.name.endswith(".yml") is False: + path = path / "challenge.yml" + + click.secho(f"Found {path}") + challenge = load_challenge(path) + click.secho(f'Loaded {challenge["name"]}', fg="yellow") + + click.secho(f'Verifying {challenge["name"]}', fg="yellow") + pull_challenge(challenge=challenge, create_files=create_files) + click.secho(f"Success!", fg="green") \ No newline at end of file diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index 8595e66..511e54e 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -404,3 +404,133 @@ def lint_challenge(path): exit(1) exit(0) + +def get_challenge_details(challenge_id): + s = generate_session() + r = s.get(f"/api/v1/challenges/{challenge_id}") + r.raise_for_status() + challenge = r.json()["data"] + # Remove non-yaml fields + + challenge.pop("id") + challenge.pop("type_data") + challenge.pop("view") + + # Add flags + r = s.get(f"/api/v1/challenges/{challenge_id}/flags") + r.raise_for_status() + flags = r.json()["data"] + challenge["flags"] = [f["content"] if f["type"] == "static" and f["data"] == None else { "content": f["content"], "type": f["type"], "data": f["data"] } for f in flags] + + # Add tags + r = s.get(f"/api/v1/challenges/{challenge_id}/tags") + r.raise_for_status() + tags = r.json()["data"] + challenge["tags"] = [t["value"] for t in tags] + + # Add hints + r = s.get(f"/api/v1/challenges/{challenge_id}/hints") + r.raise_for_status() + hints = r.json()["data"] + challenge["hints"] = [{ "content": h["content"], "cost": h["cost"] } if h["cost"] > 0 else h["content"] for h in hints] + + # Add topics + r = s.get(f"/api/v1/challenges/{challenge_id}/topics") + r.raise_for_status() + topics = r.json()["data"] + challenge["topics"] = [t["value"] for t in topics] + + # Add requirements + r = s.get(f"/api/v1/challenges/{challenge_id}/requirements") + r.raise_for_status() + requirements = r.json()["data"].get("prerequisites", []) + if len(requirements) > 0: + # Prefer challenge names over IDs + r = s.get("/api/v1/challenges") + r.raise_for_status() + challenges = r.json()["data"] + challenge["requirements"] = [c["name"] for c in challenges if c["id"] in requirements] + + return challenge + +def verify_challenge(challenge, verify_contents=False): + """ + Verify that the challenge.yml matches the remote challenge if one exists with the same name + """ + s = generate_session() + + installed_challenges = load_installed_challenges() + for c in installed_challenges: + if c["name"] == challenge["name"]: + challenge_id = c["id"] + break + else: + return + + remote_challenge = get_challenge_details(challenge_id) + for key in remote_challenge: + # Special validation needed for files + if key == "files": + # Get base file name of challenge files + local_files = {Path(f).name : f for f in challenge[key]} + + for f in remote_challenge["files"]: + # Get base file name + f_base = f.split("/")[-1] + if f_base not in local_files: + raise Exception(f"Remote challenge has file {f_base} that is not present locally") + else: + if verify_contents: + # Download remote file and compare contents + req = s.get(f"files/{f}") + req.raise_for_status() + remote_file = req.content + local_file = Path(challenge.directory, local_files[f_base]).read_bytes() + if remote_file != local_file: + raise Exception(f"Remote challenge file {f_base} does not match local file") + + if key not in challenge: + raise Exception(f"Missing field {key} in challenge.yml") + + if challenge[key] != remote_challenge[key]: + raise Exception(f"Field {key} in challenge.yml does not match remote challenge") + +def pull_challenge(challenge, create_files=False): + """ + Rewrite challenge.yml and local files to match the remote challenge + """ + s = generate_session() + installed_challenges = load_installed_challenges() + for c in installed_challenges: + if c["name"] == challenge["name"]: + challenge_id = c["id"] + break + else: + return + + details = get_challenge_details(challenge_id) + local_files = {Path(f).name : f for f in challenge["files"]} + # Update files + remote_files = details["files"] + for i, f in enumerate(remote_files): + # Get base file name + f_base = f.split("/")[-1] + if f_base not in local_files and create_files: + # Download remote file and save locally + req = s.get(f"files/{f}") + req.raise_for_status() + Path(challenge.directory, f_base).write_bytes(req.content) + details["files"].append(f_base) + + else: + # Download remote file and replace local file + req = s.get(f"files/{f}") + req.raise_for_status() + remote_file = req.content + local_file = Path(challenge.directory, local_files[f_base]) + local_file.write_bytes(remote_file) + details["files"][i] = local_files[f_base] + + # Update challenge.yml + with open(Path(challenge.directory, "challenge.yml"), "w") as f: + yaml.dump(details, f) \ No newline at end of file From 0e673173b96295bd7c8c1ef2721e89db3d9b9258 Mon Sep 17 00:00:00 2001 From: Peter Stenger Date: Wed, 16 Nov 2022 23:04:09 -0600 Subject: [PATCH 2/9] test: ensure everything works --- README.md | 11 +++-- ctfcli/cli/challenges.py | 12 ++--- ctfcli/utils/challenge.py | 101 +++++++++++++++++++++++++++----------- 3 files changed, 87 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 44027ff..2eb2305 100644 --- a/README.md +++ b/README.md @@ -87,19 +87,24 @@ Success! If, at any point, changes may have been made to a challenge through the CTFd UI by an admin. To verify that your challenge.yml file matches remote contents, you can use: ``` -❯ ctf challenge verify [challenge.yml | DIRECTORY] [--verify-contents] +❯ ctf challenge verify [challenge.yml | DIRECTORY] [--verify-files] [--verify-defaults] ``` -If the `--verify-contents` flag is set, challenge files will be downloaded and the binary contents will be compared. +If the `--verify-files` flag is set, challenge files will be downloaded and the binary files will be compared. +If the `--verify-defaults` flag is set, challenge files will be verified to make sure they include default optional keys present on CTFd. If you want to pull down the latest version of the challenge, and its challenge files, you can use: ``` -❯ ctf challenge verify [challenge.yml | DIRECTORY] [--create-files] +❯ ctf challenge verify [challenge.yml | DIRECTORY] [--update_files] [--create-files] [--create_defaults] ``` +If the `--update_files` flag is set, the latest version of every file will be redownloaded from CTFd. + If the `--create-files` flag is set, any new files added to through the CTFd UI will be downloaded to the same directory as the `challenge.yml` file. +If the `--create-defaults` flag is set, any optional default values will be added to the `challenge.yml`. + **This is a destructive action! It will overwrite the local version of `challenge.yml` with the version on CTFd!** # Challenge Templates diff --git a/ctfcli/cli/challenges.py b/ctfcli/cli/challenges.py index 5d1a2b6..e5c4f2a 100644 --- a/ctfcli/cli/challenges.py +++ b/ctfcli/cli/challenges.py @@ -417,7 +417,7 @@ def healthcheck(self, challenge): ) sys.exit(0) - def verify(self, challenge=None, verify_contents=False): + def verify(self, challenge=None, verify_files=False, verify_defaults=False): if challenge is None: # Get all challenges if not specifying a challenge config = load_config() @@ -436,10 +436,10 @@ def verify(self, challenge=None, verify_contents=False): click.secho(f'Loaded {challenge["name"]}', fg="yellow") click.secho(f'Verifying {challenge["name"]}', fg="yellow") - verify_challenge(challenge=challenge, verify_contents=verify_contents) - click.secho(f"Success!", fg="green") + verify_challenge(challenge=challenge, verify_files=verify_files, verify_defaults=verify_defaults) + click.secho("Success!", fg="green") - def pull(self, challenge=None, create_files=False): + def pull(self, challenge=None, update_files=False, create_files=False, create_defaults=False): if challenge is None: # Get all challenges if not specifying a challenge config = load_config() @@ -458,5 +458,5 @@ def pull(self, challenge=None, create_files=False): click.secho(f'Loaded {challenge["name"]}', fg="yellow") click.secho(f'Verifying {challenge["name"]}', fg="yellow") - pull_challenge(challenge=challenge, create_files=create_files) - click.secho(f"Success!", fg="green") \ No newline at end of file + pull_challenge(challenge=challenge, update_files=update_files, create_files=create_files, create_defaults=create_defaults) + click.secho("Success!", fg="green") \ No newline at end of file diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index 511e54e..ae83e06 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -407,7 +407,7 @@ def lint_challenge(path): def get_challenge_details(challenge_id): s = generate_session() - r = s.get(f"/api/v1/challenges/{challenge_id}") + r = s.get(f"/api/v1/challenges/{challenge_id}", json=True) r.raise_for_status() challenge = r.json()["data"] # Remove non-yaml fields @@ -415,45 +415,65 @@ def get_challenge_details(challenge_id): challenge.pop("id") challenge.pop("type_data") challenge.pop("view") + challenge.pop("solves") + challenge.pop("solved_by_me") + + # Normalize fields to ctfcli format + challenge['attempts'] = challenge['max_attempts'] + challenge.pop("max_attempts") + challenge['description'] = challenge['description'].replace('\r\n', '\n') # Add flags - r = s.get(f"/api/v1/challenges/{challenge_id}/flags") + r = s.get(f"/api/v1/challenges/{challenge_id}/flags", json=True) r.raise_for_status() flags = r.json()["data"] challenge["flags"] = [f["content"] if f["type"] == "static" and f["data"] == None else { "content": f["content"], "type": f["type"], "data": f["data"] } for f in flags] # Add tags - r = s.get(f"/api/v1/challenges/{challenge_id}/tags") + r = s.get(f"/api/v1/challenges/{challenge_id}/tags", json=True) r.raise_for_status() tags = r.json()["data"] challenge["tags"] = [t["value"] for t in tags] # Add hints - r = s.get(f"/api/v1/challenges/{challenge_id}/hints") + r = s.get(f"/api/v1/challenges/{challenge_id}/hints", json=True) r.raise_for_status() hints = r.json()["data"] challenge["hints"] = [{ "content": h["content"], "cost": h["cost"] } if h["cost"] > 0 else h["content"] for h in hints] # Add topics - r = s.get(f"/api/v1/challenges/{challenge_id}/topics") + r = s.get(f"/api/v1/challenges/{challenge_id}/topics", json=True) r.raise_for_status() topics = r.json()["data"] challenge["topics"] = [t["value"] for t in topics] # Add requirements - r = s.get(f"/api/v1/challenges/{challenge_id}/requirements") + r = s.get(f"/api/v1/challenges/{challenge_id}/requirements", json=True) r.raise_for_status() - requirements = r.json()["data"].get("prerequisites", []) + requirements = (r.json().get("data") or {}).get("prerequisites", []) if len(requirements) > 0: # Prefer challenge names over IDs - r = s.get("/api/v1/challenges") + r = s.get("/api/v1/challenges", json=True) r.raise_for_status() challenges = r.json()["data"] challenge["requirements"] = [c["name"] for c in challenges if c["id"] in requirements] return challenge -def verify_challenge(challenge, verify_contents=False): +def is_default(key, value): + if key == "connection_info" and value == None: + return True + if key == "attempts" and value == 0: + return True + if key == "state" and value == "visible": + return True + if key == "type" and value == "standard": + return True + if key in ["tags", "hints", "topics", "requirements"] and value == []: + return True + return False + +def verify_challenge(challenge, verify_files=False, verify_defaults=False, _verify_new_files=True): """ Verify that the challenge.yml matches the remote challenge if one exists with the same name """ @@ -476,29 +496,42 @@ def verify_challenge(challenge, verify_contents=False): for f in remote_challenge["files"]: # Get base file name - f_base = f.split("/")[-1] - if f_base not in local_files: + f_base = f.split("/")[-1].split('?token=')[0] + if f_base not in local_files and _verify_new_files: raise Exception(f"Remote challenge has file {f_base} that is not present locally") else: - if verify_contents: + if verify_files: # Download remote file and compare contents - req = s.get(f"files/{f}") + req = s.get(f) req.raise_for_status() remote_file = req.content local_file = Path(challenge.directory, local_files[f_base]).read_bytes() if remote_file != local_file: raise Exception(f"Remote challenge file {f_base} does not match local file") - if key not in challenge: + elif key not in challenge: + # Ignore optional keys with default values + if is_default(key, remote_challenge[key]) and not verify_defaults: + continue + raise Exception(f"Missing field {key} in challenge.yml") - if challenge[key] != remote_challenge[key]: + elif challenge[key] != remote_challenge[key]: raise Exception(f"Field {key} in challenge.yml does not match remote challenge") -def pull_challenge(challenge, create_files=False): +def pull_challenge(challenge, update_files=False, create_files=False, create_defaults=False): """ Rewrite challenge.yml and local files to match the remote challenge """ + + # Update challenge.yml if verification shows that something changed + try: + verify_challenge(challenge, verify_files=update_files, verify_defaults=create_defaults, _verify_new_files=create_files) + click.secho(f"Challenge {challenge['name']} is already up to date", fg="green") + return + except Exception: + pass + s = generate_session() installed_challenges = load_installed_challenges() for c in installed_challenges: @@ -508,29 +541,41 @@ def pull_challenge(challenge, create_files=False): else: return - details = get_challenge_details(challenge_id) + remote_details = get_challenge_details(challenge_id) + # Ignore optional fields with default values + remote_details = {k: v for k, v in remote_details.items() if not is_default(k, v) or create_defaults} local_files = {Path(f).name : f for f in challenge["files"]} # Update files - remote_files = details["files"] - for i, f in enumerate(remote_files): + for f in remote_details["files"]: # Get base file name - f_base = f.split("/")[-1] + f_base = f.split("/")[-1].split('?token=')[0] if f_base not in local_files and create_files: + print(f"Creating file {f_base} for challenge {challenge['name']}") # Download remote file and save locally - req = s.get(f"files/{f}") + req = s.get(f) req.raise_for_status() Path(challenge.directory, f_base).write_bytes(req.content) - details["files"].append(f_base) + challenge["files"].append(f_base) - else: + elif f_base in local_files and update_files: # Download remote file and replace local file - req = s.get(f"files/{f}") + print(f"Updating file {f_base} for challenge {challenge['name']}") + req = s.get(f) req.raise_for_status() remote_file = req.content local_file = Path(challenge.directory, local_files[f_base]) local_file.write_bytes(remote_file) - details["files"][i] = local_files[f_base] - # Update challenge.yml - with open(Path(challenge.directory, "challenge.yml"), "w") as f: - yaml.dump(details, f) \ No newline at end of file + del remote_details["files"] + + print(f"Updating challenge.yml for {challenge['name']}") + + # Merge local and remote challenge.yml & Preserve local keys + order + updated_challenge = dict(challenge) + + for key in remote_details: + updated_challenge[key] = remote_details[key] + + with open(challenge.file_path, "w") as f: + yaml.dump(updated_challenge, f, sort_keys=False) + \ No newline at end of file From 16fdecea30f26fdf46f4c4699dce735152af4a80 Mon Sep 17 00:00:00 2001 From: Peter Stenger Date: Thu, 17 Nov 2022 12:27:18 -0600 Subject: [PATCH 3/9] Add ignore, improve formatting of long strings --- ctfcli/cli/challenges.py | 14 ++++++++++---- ctfcli/utils/challenge.py | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/ctfcli/cli/challenges.py b/ctfcli/cli/challenges.py index e5c4f2a..39cedf5 100644 --- a/ctfcli/cli/challenges.py +++ b/ctfcli/cli/challenges.py @@ -417,7 +417,10 @@ def healthcheck(self, challenge): ) sys.exit(0) - def verify(self, challenge=None, verify_files=False, verify_defaults=False): + def verify(self, challenge=None, ignore=(), verify_files=False, verify_defaults=False): + if isinstance(ignore, str): + ignore = (ignore,) + if challenge is None: # Get all challenges if not specifying a challenge config = load_config() @@ -436,10 +439,13 @@ def verify(self, challenge=None, verify_files=False, verify_defaults=False): click.secho(f'Loaded {challenge["name"]}', fg="yellow") click.secho(f'Verifying {challenge["name"]}', fg="yellow") - verify_challenge(challenge=challenge, verify_files=verify_files, verify_defaults=verify_defaults) + verify_challenge(challenge=challenge, ignore=ignore, verify_files=verify_files, verify_defaults=verify_defaults) click.secho("Success!", fg="green") - def pull(self, challenge=None, update_files=False, create_files=False, create_defaults=False): + def pull(self, challenge=None, ignore=(), update_files=False, create_files=False, create_defaults=False): + if isinstance(ignore, str): + ignore = (ignore,) + if challenge is None: # Get all challenges if not specifying a challenge config = load_config() @@ -458,5 +464,5 @@ def pull(self, challenge=None, update_files=False, create_files=False, create_de click.secho(f'Loaded {challenge["name"]}', fg="yellow") click.secho(f'Verifying {challenge["name"]}', fg="yellow") - pull_challenge(challenge=challenge, update_files=update_files, create_files=create_files, create_defaults=create_defaults) + pull_challenge(challenge=challenge, ignore=ignore, update_files=update_files, create_files=create_files, create_defaults=create_defaults) click.secho("Success!", fg="green") \ No newline at end of file diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index ae83e06..f9f1726 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -427,7 +427,7 @@ def get_challenge_details(challenge_id): r = s.get(f"/api/v1/challenges/{challenge_id}/flags", json=True) r.raise_for_status() flags = r.json()["data"] - challenge["flags"] = [f["content"] if f["type"] == "static" and f["data"] == None else { "content": f["content"], "type": f["type"], "data": f["data"] } for f in flags] + challenge["flags"] = [f["content"] if f["type"] == "static" and (f["data"] == None or f["data"] == "") else { "content": f["content"], "type": f["type"], "data": f["data"] } for f in flags] # Add tags r = s.get(f"/api/v1/challenges/{challenge_id}/tags", json=True) @@ -473,7 +473,7 @@ def is_default(key, value): return True return False -def verify_challenge(challenge, verify_files=False, verify_defaults=False, _verify_new_files=True): +def verify_challenge(challenge, ignore=(), verify_files=False, verify_defaults=False, _verify_new_files=True): """ Verify that the challenge.yml matches the remote challenge if one exists with the same name """ @@ -489,6 +489,8 @@ def verify_challenge(challenge, verify_files=False, verify_defaults=False, _veri remote_challenge = get_challenge_details(challenge_id) for key in remote_challenge: + if key in ignore: + continue # Special validation needed for files if key == "files": # Get base file name of challenge files @@ -519,14 +521,29 @@ def verify_challenge(challenge, verify_files=False, verify_defaults=False, _veri elif challenge[key] != remote_challenge[key]: raise Exception(f"Field {key} in challenge.yml does not match remote challenge") -def pull_challenge(challenge, update_files=False, create_files=False, create_defaults=False): +def pull_challenge(challenge, ignore=(), update_files=False, create_files=False, create_defaults=False): """ Rewrite challenge.yml and local files to match the remote challenge """ + # Prefer multi-line YAML formatting (https://stackoverflow.com/q/8640959/15261182) + def str_presenter(dumper, data): + if len(data.splitlines()) > 1 or '\n' in data: + text_list = [line.rstrip() for line in data.splitlines()] + fixed_data = "\n".join(text_list) + return dumper.represent_scalar('tag:yaml.org,2002:str', fixed_data, style='|') + elif len(data) > 80: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='>') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + + yaml.add_representer(str, str_presenter) + yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + + + # Update challenge.yml if verification shows that something changed try: - verify_challenge(challenge, verify_files=update_files, verify_defaults=create_defaults, _verify_new_files=create_files) + verify_challenge(challenge, ignore=ignore, verify_files=update_files, verify_defaults=create_defaults, _verify_new_files=create_files) click.secho(f"Challenge {challenge['name']} is already up to date", fg="green") return except Exception: @@ -566,16 +583,24 @@ def pull_challenge(challenge, update_files=False, create_files=False, create_def local_file = Path(challenge.directory, local_files[f_base]) local_file.write_bytes(remote_file) + # Remove files that are no longer present on the remote challenge + remote_cleaned_files = [f.split("/")[-1].split('?token=')[0] for f in remote_details["files"]] + challenge["files"] = [f for f in challenge["files"] if Path(f).name in remote_cleaned_files] del remote_details["files"] - + print(f"Updating challenge.yml for {challenge['name']}") # Merge local and remote challenge.yml & Preserve local keys + order updated_challenge = dict(challenge) for key in remote_details: + if key in ignore: + continue updated_challenge[key] = remote_details[key] + # Hack: remove whitespace before newlines in multiline strings + updated_challenge['description'] = updated_challenge['description'].replace('\t', '') + with open(challenge.file_path, "w") as f: - yaml.dump(updated_challenge, f, sort_keys=False) + yaml.dump(updated_challenge, f, allow_unicode=True, sort_keys=False) \ No newline at end of file From 0a02509d33fff257359d6350ca551d4b27502101 Mon Sep 17 00:00:00 2001 From: Peter Stenger Date: Thu, 17 Nov 2022 12:46:04 -0600 Subject: [PATCH 4/9] use inline-fold more --- ctfcli/utils/challenge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index f9f1726..0a903c5 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -532,7 +532,7 @@ def str_presenter(dumper, data): fixed_data = "\n".join(text_list) return dumper.represent_scalar('tag:yaml.org,2002:str', fixed_data, style='|') elif len(data) > 80: - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='>') + return dumper.represent_scalar('tag:yaml.org,2002:str', data.rstrip(), style='>') return dumper.represent_scalar('tag:yaml.org,2002:str', data) yaml.add_representer(str, str_presenter) From 4a2bdc19dd6fecf775aeb87f97a0d8f83d42cb2b Mon Sep 17 00:00:00 2001 From: Peter Stenger Date: Thu, 17 Nov 2022 17:21:50 -0600 Subject: [PATCH 5/9] normalize extra fields for dynamic challenges --- ctfcli/utils/challenge.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index 0a903c5..184617e 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -422,6 +422,11 @@ def get_challenge_details(challenge_id): challenge['attempts'] = challenge['max_attempts'] challenge.pop("max_attempts") challenge['description'] = challenge['description'].replace('\r\n', '\n') + + for key in ['initial', 'decay', 'minimum']: + if key in challenge: + challenge['extra'][key] = challenge[key] + challenge.pop(key) # Add flags r = s.get(f"/api/v1/challenges/{challenge_id}/flags", json=True) From 892181b92c2cccd26f2458a97d70d598d74fce04 Mon Sep 17 00:00:00 2001 From: Pete Stenger Date: Tue, 29 Nov 2022 13:08:51 -0600 Subject: [PATCH 6/9] fix bug if files key not defined --- ctfcli/utils/challenge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index 184617e..2fb7496 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -566,7 +566,7 @@ def str_presenter(dumper, data): remote_details = get_challenge_details(challenge_id) # Ignore optional fields with default values remote_details = {k: v for k, v in remote_details.items() if not is_default(k, v) or create_defaults} - local_files = {Path(f).name : f for f in challenge["files"]} + local_files = {Path(f).name : f for f in challenge.get("files",[])} # Update files for f in remote_details["files"]: # Get base file name @@ -590,7 +590,7 @@ def str_presenter(dumper, data): # Remove files that are no longer present on the remote challenge remote_cleaned_files = [f.split("/")[-1].split('?token=')[0] for f in remote_details["files"]] - challenge["files"] = [f for f in challenge["files"] if Path(f).name in remote_cleaned_files] + challenge["files"] = [f for f in challenge.get("files",[]) if Path(f).name in remote_cleaned_files] del remote_details["files"] print(f"Updating challenge.yml for {challenge['name']}") From fc286b8f9b792c5c6c498772955b31269aa851ea Mon Sep 17 00:00:00 2001 From: Pete Stenger Date: Tue, 29 Nov 2022 13:22:19 -0600 Subject: [PATCH 7/9] prefer ordering keys as defined in spec --- ctfcli/utils/challenge.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index 2fb7496..af7909e 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from pathlib import Path import subprocess @@ -595,15 +596,25 @@ def str_presenter(dumper, data): print(f"Updating challenge.yml for {challenge['name']}") + # Prefer ordering remote details as described by spec + preferred_order = ["name", "category", "description", \ + "value", "type", "connection_info", "attempts", \ + "flags", "topics", "tags", "files", "hints", "requirements", "state"] + # Merge local and remote challenge.yml & Preserve local keys + order updated_challenge = dict(challenge) + # Add all preferred keys + for key in preferred_order: + if key in remote_details and key not in ignore: + updated_challenge[key] = remote_details[key] + + # Add remaining keys for key in remote_details: - if key in ignore: - continue - updated_challenge[key] = remote_details[key] + if key not in preferred_order and key not in ignore: + updated_challenge[key] = remote_details[key] - # Hack: remove whitespace before newlines in multiline strings + # Hack: remove tabs in multiline strings updated_challenge['description'] = updated_challenge['description'].replace('\t', '') with open(challenge.file_path, "w") as f: From ec9d3c070df2cc2a6adeff70be26cca719ea8056 Mon Sep 17 00:00:00 2001 From: Pete Stenger Date: Tue, 29 Nov 2022 13:28:42 -0600 Subject: [PATCH 8/9] fig another files bug --- ctfcli/utils/challenge.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index af7909e..b788d0b 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -578,6 +578,8 @@ def str_presenter(dumper, data): req = s.get(f) req.raise_for_status() Path(challenge.directory, f_base).write_bytes(req.content) + if "files" not in challenge: + challenge["files"] = [] challenge["files"].append(f_base) elif f_base in local_files and update_files: From 1c7c4d40f6dd4a30e67f795b9c161c80a77b792d Mon Sep 17 00:00:00 2001 From: Pete Stenger Date: Tue, 29 Nov 2022 13:47:28 -0600 Subject: [PATCH 9/9] don't ignore remote default keys if the value differs from local files --- ctfcli/utils/challenge.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ctfcli/utils/challenge.py b/ctfcli/utils/challenge.py index b788d0b..fbaa6b0 100644 --- a/ctfcli/utils/challenge.py +++ b/ctfcli/utils/challenge.py @@ -565,8 +565,7 @@ def str_presenter(dumper, data): return remote_details = get_challenge_details(challenge_id) - # Ignore optional fields with default values - remote_details = {k: v for k, v in remote_details.items() if not is_default(k, v) or create_defaults} + local_files = {Path(f).name : f for f in challenge.get("files",[])} # Update files for f in remote_details["files"]: @@ -606,15 +605,24 @@ def str_presenter(dumper, data): # Merge local and remote challenge.yml & Preserve local keys + order updated_challenge = dict(challenge) + # Ignore optional fields with default values + remote_details_updates = {} + for k, v in remote_details.items(): + # If the key value changed, we want to update it + if k in challenge and challenge[k] != v: + remote_details_updates[k] = v + elif not is_default(k, v) or create_defaults: + remote_details_updates[k] = v + # Add all preferred keys for key in preferred_order: - if key in remote_details and key not in ignore: - updated_challenge[key] = remote_details[key] + if key in remote_details_updates and key not in ignore: + updated_challenge[key] = remote_details_updates[key] # Add remaining keys - for key in remote_details: + for key in remote_details_updates: if key not in preferred_order and key not in ignore: - updated_challenge[key] = remote_details[key] + updated_challenge[key] = remote_details_updates[key] # Hack: remove tabs in multiline strings updated_challenge['description'] = updated_challenge['description'].replace('\t', '')