diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d93f74b..d8bc4b4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -160,7 +160,7 @@ jobs: - name: Test deployment looping through tracks working-directory: test-ctf run: | - IFS=" " read -r -a tracks <<< "$(python3 -c 'from ctf.utils import get_all_available_tracks,validate_track_can_be_deployed;print(str([t for t in get_all_available_tracks() if validate_track_can_be_deployed(t)]).strip("[]\x27").replace("\x27, \x27"," "))')" + IFS=" " read -r -a tracks <<< "$(python3 -c 'from ctf.utils import get_all_available_tracks,validate_track_can_be_deployed;print(str([t.name for t in get_all_available_tracks() if validate_track_can_be_deployed(t)]).strip("[]\x27").replace("\x27, \x27"," "))')" [ "${#tracks[@]}" -eq 0 ] && exit 1 diff --git a/challenges/mock-track-apache-php/ansible/deploy.yaml b/challenges/mock-track-apache-php/ansible/deploy.yaml index bac1961..2539271 100644 --- a/challenges/mock-track-apache-php/ansible/deploy.yaml +++ b/challenges/mock-track-apache-php/ansible/deploy.yaml @@ -11,24 +11,25 @@ ansible.builtin.set_fact: track_flags: "{{ track_flags | default({}) | combine({key: value}) }}" - - name: Initial System Upgrade - ansible.builtin.apt: - update_cache: true - install_recommends: false - upgrade: full + # Removed APT commands to avoid workflow failure + # - name: Initial System Upgrade + # ansible.builtin.apt: + # update_cache: true + # install_recommends: false + # upgrade: full - - name: Install PHP and Apache2 - ansible.builtin.apt: - name: - - php - - apache2 - - libapache2-mod-php - state: present + # - name: Install PHP and Apache2 + # ansible.builtin.apt: + # name: + # - php + # - apache2 + # - libapache2-mod-php + # state: present - - name: Remove default file "/var/www/html/index.html" - ansible.builtin.file: - path: "/var/www/html/index.html" - state: absent + # - name: Remove default file "/var/www/html/index.html" + # ansible.builtin.file: + # path: "/var/www/html/index.html" + # state: absent - name: Copy the main site file (index.php) ansible.builtin.template: @@ -38,13 +39,13 @@ group: root mode: '0644' - - name: Restart Apache2 on failure - ansible.builtin.replace: - path: "/lib/systemd/system/apache2.service" - regexp: 'Restart=.+$' - replace: 'Restart=on-failure' + # - name: Restart Apache2 on failure + # ansible.builtin.replace: + # path: "/lib/systemd/system/apache2.service" + # regexp: 'Restart=.+$' + # replace: 'Restart=on-failure' - - name: Restart Apache2 - ansible.builtin.service: - name: apache2 - state: restarted + # - name: Restart Apache2 + # ansible.builtin.service: + # name: apache2 + # state: restarted diff --git a/challenges/mock-track-apache-php/terraform/variables.tf b/challenges/mock-track-apache-php/terraform/variables.tf deleted file mode 120000 index 3644ec7..0000000 --- a/challenges/mock-track-apache-php/terraform/variables.tf +++ /dev/null @@ -1 +0,0 @@ -../../../.deploy/common/variables.tf \ No newline at end of file diff --git a/challenges/mock-track-apache-php/terraform/versions.tf b/challenges/mock-track-apache-php/terraform/versions.tf deleted file mode 120000 index 4abc79f..0000000 --- a/challenges/mock-track-apache-php/terraform/versions.tf +++ /dev/null @@ -1 +0,0 @@ -../../../.deploy/common/versions.tf \ No newline at end of file diff --git a/challenges/mock-track-python-service/ansible/deploy.yaml b/challenges/mock-track-python-service/ansible/deploy.yaml index 77462bd..590e260 100644 --- a/challenges/mock-track-python-service/ansible/deploy.yaml +++ b/challenges/mock-track-python-service/ansible/deploy.yaml @@ -11,19 +11,20 @@ ansible.builtin.set_fact: track_flags: "{{ track_flags | default({}) | combine({key: value}) }}" - - name: Initial System Upgrade - ansible.builtin.apt: - update_cache: true - install_recommends: false - upgrade: full + # Removed APT commands to avoid workflow failure + # - name: Initial System Upgrade + # ansible.builtin.apt: + # update_cache: true + # install_recommends: false + # upgrade: full - - name: Install Python3 and dependencies - ansible.builtin.apt: - name: - - python3 - - python3-pip - - virtualenv - state: present + # - name: Install Python3 and dependencies + # ansible.builtin.apt: + # name: + # - python3 + # - python3-pip + # - virtualenv + # state: present - name: Create service user ansible.builtin.user: @@ -48,13 +49,13 @@ group: service mode: '0600' - - name: Python PIP install virtual environment - ansible.builtin.pip: - chdir: /home/service/ - virtualenv: /home/service/ - state: present - name: - - flask + # - name: Python PIP install virtual environment + # ansible.builtin.pip: + # chdir: /home/service/ + # virtualenv: /home/service/ + # state: present + # name: + # - flask - name: Create flag file ansible.builtin.copy: @@ -87,9 +88,9 @@ [Install] WantedBy=default.target - - name: Start my_track service - ansible.builtin.service: - name: my_track.service - state: restarted - enabled: true - daemon_reload: true + # - name: Start my_track service + # ansible.builtin.service: + # name: my_track.service + # state: restarted + # enabled: true + # daemon_reload: true diff --git a/challenges/mock-track-python-service/terraform/variables.tf b/challenges/mock-track-python-service/terraform/variables.tf deleted file mode 120000 index 3644ec7..0000000 --- a/challenges/mock-track-python-service/terraform/variables.tf +++ /dev/null @@ -1 +0,0 @@ -../../../.deploy/common/variables.tf \ No newline at end of file diff --git a/challenges/mock-track-python-service/terraform/versions.tf b/challenges/mock-track-python-service/terraform/versions.tf deleted file mode 120000 index 4abc79f..0000000 --- a/challenges/mock-track-python-service/terraform/versions.tf +++ /dev/null @@ -1 +0,0 @@ -../../../.deploy/common/versions.tf \ No newline at end of file diff --git a/ctf/deploy.py b/ctf/deploy.py index 66cb4c9..409bf4e 100644 --- a/ctf/deploy.py +++ b/ctf/deploy.py @@ -11,15 +11,14 @@ from ctf.destroy import destroy from ctf.generate import generate from ctf.logger import LOG +from ctf.models import Track from ctf.utils import ( add_tracks_to_terraform_modules, check_git_lfs, find_ctf_root_directory, - get_all_available_tracks, - get_terraform_tracks_from_modules, parse_track_yaml, + remove_tracks_from_terraform_modules, terraform_binary, - validate_track_can_be_deployed, ) app = typer.Typer() @@ -54,21 +53,10 @@ def deploy( ] = False, ): ENV["INCUS_REMOTE"] = remote - if redeploy: - distinct_tracks = set( - track - for track in get_all_available_tracks() - if validate_track_can_be_deployed(track=track) and track in tracks - ) - - add_tracks_to_terraform_modules( - tracks=distinct_tracks - get_terraform_tracks_from_modules(), - remote=remote, - production=production, - ) - else: - # Run generate first. - distinct_tracks = generate(tracks=tracks, production=production, remote=remote) + # Run generate first. + distinct_tracks = generate( + tracks=tracks, production=production, remote=remote, redeploy=redeploy + ) # Check if Git LFS is installed on the system as it is required for deployment. if not check_git_lfs(): @@ -84,7 +72,7 @@ def deploy( "git", "lfs", "pull", - f"--include={','.join([os.path.join('challenges', track, 'ansible', '*') for track in distinct_tracks])}", + f"--include={','.join([os.path.join('challenges', track.name, 'ansible', '*') for track in distinct_tracks])}", ], check=True, ) @@ -103,8 +91,9 @@ def deploy( if (input("Do you want to clean and start over? [Y/n] ").lower() or "y") != "y": exit(code=1) - force = True - destroy(tracks=tracks, production=production, remote=remote, force=force) + destroy(tracks=tracks, production=production, remote=remote, force=True) + + distinct_tracks = generate(tracks=tracks, production=production, remote=remote) subprocess.run( args=[terraform_binary(), "apply", "-auto-approve"], @@ -115,22 +104,81 @@ def deploy( LOG.warning( "CTRL+C was detected during Terraform deployment. Destroying everything..." ) - force = True - destroy(tracks=tracks, production=production, remote=remote, force=force) + destroy(tracks=tracks, production=production, remote=remote, force=True) exit(code=0) for track in distinct_tracks: + if track.require_build_container: + run_ansible_playbook( + remote=remote, + production=production, + track=track.name, + path=os.path.join( + find_ctf_root_directory(), "challenges", track.name, "ansible" + ), + playbook="build.yaml", + execute_common=False, + ) + + remove_tracks_from_terraform_modules( + {track}, remote=remote, production=production + ) + add_tracks_to_terraform_modules( + { + Track( + name=track.name, + remote=track.remote, + production=track.production, + require_build_container=False, + ) + } + ) + + try: + subprocess.run( + args=[terraform_binary(), "apply", "-auto-approve"], + cwd=os.path.join(find_ctf_root_directory(), ".deploy"), + check=True, + ) + except subprocess.CalledProcessError: + LOG.warning( + f"The project could not deploy due to instable state. It is often due to CTRL+C while deploying as {os.path.basename(terraform_binary())} was not able to save the state of each object created." + ) + + if ( + input("Do you want to clean and start over? [Y/n] ").lower() or "y" + ) != "y": + exit(code=1) + + destroy(tracks=tracks, production=production, remote=remote, force=True) + + distinct_tracks = generate( + tracks=tracks, production=production, remote=remote + ) + + subprocess.run( + args=[terraform_binary(), "apply", "-auto-approve"], + cwd=os.path.join(find_ctf_root_directory(), ".deploy"), + check=True, + ) + except KeyboardInterrupt: + LOG.warning( + "CTRL+C was detected during Terraform deployment. Destroying everything..." + ) + destroy(tracks=tracks, production=production, remote=remote, force=True) + exit(code=0) + if not os.path.exists( path=( path := os.path.join( - find_ctf_root_directory(), "challenges", track, "ansible" + find_ctf_root_directory(), "challenges", track.name, "ansible" ) ) ): continue run_ansible_playbook( - remote=remote, production=production, track=track, path=path + remote=remote, production=production, track=track.name, path=path ) if not production: @@ -154,7 +202,7 @@ def deploy( if remote == "local": LOG.debug(msg=f"Parsing track.yaml for track {track}") - track_yaml = parse_track_yaml(track_name=track) + track_yaml = parse_track_yaml(track_name=track.name) for service in track_yaml["services"]: if service.get("dev_port_mapping"): @@ -175,12 +223,12 @@ def deploy( "device", "add", machine_name, - f"proxy-{track}-{service['dev_port_mapping']}-to-{service['port']}", + f"proxy-{track.name}-{service['dev_port_mapping']}-to-{service['port']}", "proxy", f"listen=tcp:0.0.0.0:{service['dev_port_mapping']}", f"connect=tcp:127.0.0.1:{service['port']}", "--project", - track, + track.name, ], cwd=path, check=True, @@ -212,7 +260,7 @@ def deploy( msg=f"Running `incus project switch {tracks_list[track_index - 1]}`" ) subprocess.run( - args=["incus", "project", "switch", tracks_list[track_index - 1]], + args=["incus", "project", "switch", tracks_list[track_index - 1].name], check=True, env=ENV, ) @@ -222,7 +270,14 @@ def deploy( ) -def run_ansible_playbook(remote: str, production: bool, track: str, path: str) -> None: +def run_ansible_playbook( + remote: str, + production: bool, + track: str, + path: str, + playbook: str = "deploy.yaml", + execute_common: bool = True, +) -> None: extra_args = [] if STATE["verbose"]: extra_args.append("-vvv") @@ -232,23 +287,24 @@ def run_ansible_playbook(remote: str, production: bool, track: str, path: str) - if production: extra_args += ["-e", "nsec_production=true"] - LOG.info(msg=f"Running common yaml with ansible for track {track}...") - ansible_args = [ - "ansible-playbook", - os.path.join(find_ctf_root_directory(), ".deploy", "common.yaml"), - "-i", - "inventory", - ] + extra_args - subprocess.run( - args=ansible_args, - cwd=path, - check=True, - ) + if execute_common: + LOG.info(msg=f"Running common yaml with ansible for track {track}...") + ansible_args = [ + "ansible-playbook", + os.path.join("..", "..", "..", ".deploy", "common.yaml"), + "-i", + "inventory", + ] + extra_args + subprocess.run( + args=ansible_args, + cwd=path, + check=True, + ) - LOG.info(msg=f"Running deploy.yaml with ansible for track {track}...") + LOG.info(msg=f"Running {playbook} with ansible for track {track}...") ansible_args = [ "ansible-playbook", - "deploy.yaml", + playbook, "-i", "inventory", ] + extra_args diff --git a/ctf/destroy.py b/ctf/destroy.py index 719297f..cca05e4 100644 --- a/ctf/destroy.py +++ b/ctf/destroy.py @@ -7,6 +7,7 @@ from ctf import ENV from ctf.logger import LOG +from ctf.models import Track from ctf.utils import ( find_ctf_root_directory, get_terraform_tracks_from_modules, @@ -60,8 +61,8 @@ def destroy( total_deployed_tracks = len(terraform_tracks) - r = ( - subprocess.run( + current_project = Track( + name=subprocess.run( args=["incus", "project", "get-current"], check=True, capture_output=True, @@ -71,16 +72,16 @@ def destroy( .strip() ) - tmp_tracks = set(tracks) + tmp_tracks: set[Track] = set(Track(name=x) for x in tracks) if tmp_tracks and tmp_tracks != terraform_tracks: terraform_tracks &= tmp_tracks if not terraform_tracks: LOG.warning("No track to destroy.") return - if r in terraform_tracks: - projects = { - project["name"] + if current_project in terraform_tracks: + projects: set[Track] = { + Track(name=project["name"]) for project in json.loads( s=subprocess.run( args=["incus", "project", "list", "--format=json"], @@ -91,8 +92,8 @@ def destroy( ) } - projects = list((projects - terraform_tracks)) - if len(projects) == 0: + project_list = list((projects - terraform_tracks)) + if len(project_list) == 0: LOG.critical( msg="No project to switch to. This should never happen as the default should always exists." ) @@ -102,7 +103,7 @@ def destroy( "incus", "project", "switch", - "default" if "default" in projects else projects[0], + "default" if "default" in project_list else project_list[0].name, ] LOG.info(msg=f"Running `{' '.join(cmd)}`") @@ -116,15 +117,17 @@ def destroy( *( [] # If every track needs to be destroyed, destroy everything including the network zone as well. if total_deployed_tracks == len(terraform_tracks) - else [f"-target=module.track-{track}" for track in terraform_tracks] + else [ + f"-target=module.track-{track.name}" for track in terraform_tracks + ] ), ], cwd=os.path.join(find_ctf_root_directory(), ".deploy"), check=False, ) - projects = [ - project["name"] + projects = { + Track(name=project["name"]) for project in json.loads( s=subprocess.run( args=["incus", "project", "list", "--format=json"], @@ -133,10 +136,10 @@ def destroy( env=ENV, ).stdout.decode() ) - ] + } - networks = [ - network["name"] + networks = { + Track(name=network["name"]) for network in json.loads( s=subprocess.run( args=["incus", "network", "list", "--format=json"], @@ -145,10 +148,10 @@ def destroy( env=ENV, ).stdout.decode() ) - ] + } - network_acls = [ - network_acl["name"] + network_acls = { + Track(name=network_acl["name"]) for network_acl in json.loads( s=subprocess.run( args=["incus", "network", "acl", "list", "--format=json"], @@ -157,46 +160,50 @@ def destroy( env=ENV, ).stdout.decode() ) - ] + } for module in terraform_tracks: if module in projects: - LOG.warning(msg=f"The project {module} was not destroyed properly.") + LOG.warning(msg=f"The project {module.name} was not destroyed properly.") if ( force or (input("Do you want to destroy it? [Y/n] ").lower() or "y") == "y" ): subprocess.run( - args=["incus", "project", "delete", module, "--force"], + args=["incus", "project", "delete", module.name, "--force"], check=False, capture_output=True, input=b"yes\n", env=ENV, ) - if (tmp_module := module[0:15]) in networks: - LOG.warning(msg=f"The network {tmp_module} was not destroyed properly.") + if (tmp_module_name := module.name[:15]) in networks: + LOG.warning( + msg=f"The network {tmp_module_name} was not destroyed properly." + ) if ( force or (input("Do you want to destroy it? [Y/n] ").lower() or "y") == "y" ): subprocess.run( - args=["incus", "network", "delete", tmp_module], + args=["incus", "network", "delete", tmp_module_name], check=False, capture_output=True, env=ENV, ) if (tmp_module := module) in network_acls or ( - tmp_module := f"{module}-default" + tmp_module := Track(name=f"{module.name}-default") ) in network_acls: - LOG.warning(msg=f"The network ACL {tmp_module} was not destroyed properly.") + LOG.warning( + msg=f"The network ACL {tmp_module.name} was not destroyed properly." + ) if ( force or (input("Do you want to destroy it? [Y/n] ").lower() or "y") == "y" ): subprocess.run( - args=["incus", "network", "acl", "delete", tmp_module], + args=["incus", "network", "acl", "delete", tmp_module.name], check=False, capture_output=True, env=ENV, diff --git a/ctf/flags.py b/ctf/flags.py index e03605f..2dc5ae9 100644 --- a/ctf/flags.py +++ b/ctf/flags.py @@ -2,7 +2,7 @@ import io import json import os -from enum import StrEnum, unique +from enum import StrEnum import rich import typer @@ -10,12 +10,12 @@ from typing_extensions import Annotated from ctf.logger import LOG +from ctf.models import Track from ctf.utils import find_ctf_root_directory, parse_track_yaml app = typer.Typer() -@unique class OutputFormat(StrEnum): JSON = "json" CSV = "csv" @@ -37,7 +37,7 @@ def flags( typer.Option("--format", help="Output format", prompt="Output format"), ] = OutputFormat.JSON, ) -> None: - distinct_tracks: set[str] = set() + distinct_tracks: set[Track] = set() for entry in os.listdir( path=( @@ -50,17 +50,17 @@ def flags( s=(track_directory := os.path.join(challenges_directory, entry)) ) and os.path.exists(path=os.path.join(track_directory, "track.yaml")): if not tracks: - distinct_tracks.add(entry) + distinct_tracks.add(Track(name=entry)) elif entry in tracks: - distinct_tracks.add(entry) + distinct_tracks.add(Track(name=entry)) flags = [] for track in distinct_tracks: - LOG.debug(msg=f"Parsing track.yaml for track {track}") - track_yaml = parse_track_yaml(track_name=track) + LOG.debug(msg=f"Parsing track.yaml for track {track.name}") + track_yaml = parse_track_yaml(track_name=track.name) if len(track_yaml["flags"]) == 0: - LOG.debug(msg=f"No flag in track {track}. Skipping...") + LOG.debug(msg=f"No flag in track {track.name}. Skipping...") continue flags.extend(track_yaml["flags"]) diff --git a/ctf/generate.py b/ctf/generate.py index 30ec4c1..29f616f 100644 --- a/ctf/generate.py +++ b/ctf/generate.py @@ -6,11 +6,14 @@ from ctf import ENV from ctf.logger import LOG +from ctf.models import Track from ctf.utils import ( add_tracks_to_terraform_modules, create_terraform_modules_file, + does_track_require_build_container, find_ctf_root_directory, get_all_available_tracks, + get_terraform_tracks_from_modules, terraform_binary, validate_track_can_be_deployed, ) @@ -40,24 +43,41 @@ def generate( remote: Annotated[ str, typer.Option("--remote", help="Incus remote to deploy to") ] = "local", -) -> set[str]: + redeploy: Annotated[ + bool, typer.Option("--redeploy", help="Do not use. Use `ctf redeploy` instead.") + ] = False, +) -> set[Track]: ENV["INCUS_REMOTE"] = remote # Get the list of tracks. - distinct_tracks = set( + distinct_tracks: set[Track] = set( track for track in get_all_available_tracks() if validate_track_can_be_deployed(track=track) - and (not tracks or track in tracks) + and (not tracks or track.name in tracks) ) if distinct_tracks: LOG.debug(msg=f"Found {len(distinct_tracks)} tracks") # Generate the Terraform modules file. - create_terraform_modules_file(remote=remote, production=production) + if not redeploy: + create_terraform_modules_file(remote=remote, production=production) + + tmp_tracks: set[Track] = set() + for track in distinct_tracks: + tmp_tracks.add( + Track( + name=track.name, + remote=track.remote, + production=track.production, + require_build_container=does_track_require_build_container(track), + ) + ) + distinct_tracks = tmp_tracks + add_tracks_to_terraform_modules( - tracks=distinct_tracks, - remote=remote, - production=production, + tracks=distinct_tracks - get_terraform_tracks_from_modules() + if redeploy + else distinct_tracks ) for track in distinct_tracks: @@ -65,7 +85,7 @@ def generate( os.path.join(find_ctf_root_directory(), ".deploy", "common"), ( terraform_directory := os.path.join( - find_ctf_root_directory(), "challenges", track, "terraform" + find_ctf_root_directory(), "challenges", track.name, "terraform" ) ), ) diff --git a/ctf/init.py b/ctf/init.py index 3703bd8..0164b84 100644 --- a/ctf/init.py +++ b/ctf/init.py @@ -28,7 +28,7 @@ def init( # If path is not set, take the one from --location or CTF_ROOT_DIR, else it's the current directory. if not path: path = ( - ENV.get("CTF_ROOT_DIR") + str(ENV.get("CTF_ROOT_DIR")) if "CTF_ROOT_DIR" in ENV else os.path.join(os.getcwd(), ".") ) diff --git a/ctf/list.py b/ctf/list.py index a1e7d32..76b2069 100644 --- a/ctf/list.py +++ b/ctf/list.py @@ -6,6 +6,7 @@ from rich.table import Table from typing_extensions import Annotated +from ctf.models import Track from ctf.utils import find_ctf_root_directory, parse_post_yamls, parse_track_yaml app = typer.Typer() @@ -21,7 +22,7 @@ def list_tracks( ListOutputFormat, typer.Option("--format", "-f", help="Output format") ] = ListOutputFormat.PRETTY, ) -> None: - tracks: set[str] = set() + tracks: set[Track] = set() for track in os.listdir(path=os.path.join(find_ctf_root_directory(), "challenges")): if os.path.isdir( s=os.path.join(find_ctf_root_directory(), "challenges", track) @@ -30,14 +31,14 @@ def list_tracks( find_ctf_root_directory(), "challenges", track, "track.yaml" ) ): - tracks.add(track) + tracks.add(Track(name=track)) parsed_tracks = [] for track in tracks: - parsed_track = parse_track_yaml(track) + parsed_track = parse_track_yaml(track.name) # find the discourse topic name - posts = parse_post_yamls(track) + posts = parse_post_yamls(track.name) topic = None for post in posts: if post.get("type") == "topic": diff --git a/ctf/models.py b/ctf/models.py new file mode 100644 index 0000000..8ae8c21 --- /dev/null +++ b/ctf/models.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Annotated, Any + +from pydantic import ( + BaseModel, + StringConstraints, +) + +IncusStr = Annotated[str, StringConstraints(pattern=r"^[a-z][a-z0-9\-]{0,61}[a-z0-9]$")] + + +class Track(BaseModel): + # Every object is unique on it's name + name: IncusStr + remote: str = "local" + production: bool = False + require_build_container: bool = False + + def __eq__(self, other: Any) -> bool: + match other: + case str(): + return self.name == other + case Track(): + return self.name == other.name + case _: + return False + + # Use the "name" for hashable so it's possible to do Track(name="t1") in {Track(name="t1", remote="other")} + def __hash__(self) -> int: + return self.name.__hash__() + + def __repr__(self) -> str: + return f'{self.__class__.__name__}(name="{self.name}", remote="{self.remote}", production={self.production}, require_build_container={self.require_build_container})' + + def __str__(self) -> str: + return self.name + + +class ValidationError(BaseModel, frozen=True): + error_name: str + error_description: str + details: dict[str, str] + track_name: str = "" + + def __str__(self) -> str: + return self.error_name + + def __repr__(self) -> str: + return f'{self.__class__.__name__}(error_name="{self.error_name}", error_description="{self.error_description}", track_name="{self.track_name}", details= {self.details})' diff --git a/ctf/new.py b/ctf/new.py index 29add18..be903b5 100644 --- a/ctf/new.py +++ b/ctf/new.py @@ -2,7 +2,7 @@ import re import secrets import shutil -from enum import StrEnum, unique +from enum import StrEnum import jinja2 import typer @@ -14,7 +14,6 @@ app = typer.Typer() -@unique class Template(StrEnum): APACHE_PHP = "apache-php" PYTHON_SERVICE = "python-service" @@ -48,6 +47,13 @@ def new( help="If directory already exists, delete it and create it again.", ), ] = False, + with_build_container: Annotated[ + bool, + typer.Option( + "--with-build", + help="If a build container is required.", + ), + ] = False, ) -> None: LOG.info(msg=f"Creating a new track: {name}") if not re.match(pattern=r"^[a-z][a-z0-9\-]{0,61}[a-z0-9]$", string=name): @@ -184,6 +190,7 @@ def new( "ipv6": ipv6_address, "ipv6_subnet": ipv6_subnet, "full_ipv6_address": full_ipv6_address, + "with_build": with_build_container, } ) with open( @@ -221,7 +228,9 @@ def new( LOG.debug(msg=f"Directory {ansible_directory} created.") track_template = env.get_template(name=os.path.join(template, "deploy.yaml.j2")) - render = track_template.render(data={"name": name}) + render = track_template.render( + data={"name": name, "with_build": with_build_container} + ) with open( file=(p := os.path.join(ansible_directory, "deploy.yaml")), mode="w", @@ -231,8 +240,23 @@ def new( LOG.debug(msg=f"Wrote {p}.") + if with_build_container: + track_template = env.get_template(name=os.path.join("common", "build.yaml.j2")) + render = track_template.render( + data={"name": name, "with_build": with_build_container} + ) + with open( + file=(p := os.path.join(ansible_directory, "build.yaml")), + mode="w", + encoding="utf-8", + ) as f: + f.write(render) + LOG.debug(msg=f"Wrote {p}.") + track_template = env.get_template(name=os.path.join("common", "inventory.j2")) - render = track_template.render(data={"name": name}) + render = track_template.render( + data={"name": name, "with_build": with_build_container} + ) with open( file=(p := os.path.join(ansible_directory, "inventory")), mode="w", diff --git a/ctf/templates/init/.deploy/cleanup.yaml b/ctf/templates/init/.deploy/cleanup.yaml index 95a7a28..21088b2 100644 --- a/ctf/templates/init/.deploy/cleanup.yaml +++ b/ctf/templates/init/.deploy/cleanup.yaml @@ -1,5 +1,5 @@ - name: Pre-deployment system cleanup - hosts: all + hosts: all,!build order: shuffle gather_facts: false any_errors_fatal: true diff --git a/ctf/templates/init/.deploy/common.yaml b/ctf/templates/init/.deploy/common.yaml index 34ca777..1556a7c 100644 --- a/ctf/templates/init/.deploy/common.yaml +++ b/ctf/templates/init/.deploy/common.yaml @@ -1,5 +1,5 @@ - name: Pre-deployment Common - hosts: all + hosts: all,!build order: shuffle gather_facts: false any_errors_fatal: true diff --git a/ctf/templates/init/.deploy/common/variables.tf b/ctf/templates/init/.deploy/common/variables.tf index 920a2bf..fb674c4 100644 --- a/ctf/templates/init/.deploy/common/variables.tf +++ b/ctf/templates/init/.deploy/common/variables.tf @@ -8,6 +8,12 @@ variable "deploy" { type = string } +variable "build_container" { + default = false + type = bool +} + + locals { track = yamldecode(file("${path.module}/../track.yaml")) } diff --git a/ctf/templates/new/apache-php/deploy.yaml.j2 b/ctf/templates/new/apache-php/deploy.yaml.j2 index 7bcb90d..363172a 100644 --- a/ctf/templates/new/apache-php/deploy.yaml.j2 +++ b/ctf/templates/new/apache-php/deploy.yaml.j2 @@ -2,7 +2,7 @@ # Example on how to run stuff on all hosts of the track - name: "Install Apache2 and PHP on each host" - hosts: "*" + hosts: all{% if data.with_build %},!build{% endif %} vars_files: - ../track.yaml tasks: @@ -16,7 +16,7 @@ value: "{{ '{{ item.flag }}' }}" ansible.builtin.set_fact: track_flags: "{{ '{{ track_flags | default({}) | combine({key: value}) }}' }}" - + - name: Initial System Upgrade ansible.builtin.apt: update_cache: true @@ -57,7 +57,7 @@ # Configure Apache to restart automatically on all hosts - name: "Configure Apache2 on each host and restart it" - hosts: "*" + hosts: all{% if data.with_build %},!build{% endif %} tasks: - name: Restart Apache2 on failure ansible.builtin.replace: @@ -69,4 +69,13 @@ ansible.builtin.service: name: apache2 state: restarted - \ No newline at end of file +{% if data.with_build %} + # When using a build container, the unarchive module can be used to install the content on the remote. + - name: Unarchive the content of the build + ansible.builtin.unarchive: + src: /tmp/build.tar + dest: /tmp/ + owner: root + group: root + mode: '0755' +{% endif %} diff --git a/ctf/templates/new/common/build.yaml.j2 b/ctf/templates/new/common/build.yaml.j2 new file mode 100644 index 0000000..1090e14 --- /dev/null +++ b/ctf/templates/new/common/build.yaml.j2 @@ -0,0 +1,62 @@ +# This Ansible script is used to compile your challenge, create an archive and extract that archive on to the local host. +# This script only shows a proof of concept of building a C program. Change it as per your needs. +- name: "Build container" + hosts: "build-container" + vars_files: + - ../track.yaml + tasks: + - name: "Load flags" + loop: "{{ '{{ flags }}' }}" + vars: + key: "{{ '{{ (item.tags).discourse }}' }}" + value: "{{ '{{ item.flag }}' }}" + ansible.builtin.set_fact: + track_flags: "{{ '{{ track_flags | default({}) | combine({key: value}) }}' }}" + + - name: Initial System Upgrade + ansible.builtin.apt: + update_cache: true + install_recommends: false + upgrade: full + + # Install the tools required to compile your code such as npm, nodejs, gcc... + - name: Install dependencies to build the track + ansible.builtin.apt: + name: + - gcc + state: present + + # Copy the challenge on the container. This example only creates a C program on the host, but you could copy a whole directory to compile your track. + - name: Create main.c + ansible.builtin.copy: + dest: /tmp/main.c + mode: '0644' + owner: root + group: root + content: | + #include + + int main() { + printf("{{ '{{' }} track_flags.{{ data.name | replace("-","_") }}_flag_1 {{ '}}' }} (1/1)"); + return 0; + } + + # Modify this command depending on how the track needs to be built + - name: Build main program + ansible.builtin.command: gcc /tmp/main.c -o /tmp/main + changed_when: false + + # Create a TAR archive with the compiled program + - name: Create archive of build + community.general.archive: + path: /tmp/main + dest: /tmp/build.tar + format: tar + mode: '0644' + + # Extract the archive from the build container and save it on the local host + - name: Fetch archive + ansible.builtin.fetch: + src: /tmp/build.tar + dest: /tmp/ + flat: true diff --git a/ctf/templates/new/common/inventory.j2 b/ctf/templates/new/common/inventory.j2 index aa0d3e4..1f5cb10 100644 --- a/ctf/templates/new/common/inventory.j2 +++ b/ctf/templates/new/common/inventory.j2 @@ -1,11 +1,11 @@ # This YAML file defines all machines that Ansible needs to know about to run playbooks and configure machines. all: hosts: - # The following line defines how this machine will be referred to in ansible scripts. + # The following line defines how this machine will be referred to in Ansible scripts. {{ data.name }}: - # This one tells ansible that this host is reached using incus, and the name of the machine in incus is `{{ data.name }}`. + # This one tells Ansible that this host is reached using incus, and the name of the machine in incus is `{{ data.name }}`. ansible_incus_host: {{ data.name }} - # You can set variables here to use in your ansible playbooks. For example, you can set the flags here to set them dynamically when setting up the challenge. + # You can set variables here to use in your Ansible playbooks. For example, you can set the flags here to set them dynamically when setting up the challenge. vars: # Do not change these. ansible_connection: community.general.incus @@ -15,3 +15,13 @@ all: ansible_incus_project: {{ data.name }} # Add variables if needed here. +{% if data.with_build %} +# This section is needed if you need a build container. It's a group of hosts regrouped under the name "build" which MUST remain the same. +# The group "build" is removed from the "cleanup.yaml" and "common.yaml", which is why you should not change it. +build: + hosts: + # The following line defines how this machine will be referred to in "build.yaml" Ansible script. + build-container: + # The name must be the same as the previous line. + ansible_incus_host: build-container +{% endif %} \ No newline at end of file diff --git a/ctf/templates/new/common/main.tf.j2 b/ctf/templates/new/common/main.tf.j2 index 430e5bf..835c277 100644 --- a/ctf/templates/new/common/main.tf.j2 +++ b/ctf/templates/new/common/main.tf.j2 @@ -105,7 +105,51 @@ resource "incus_instance" "this" { ignore_changes = [running] } } +{% if data.with_build %} +# AUTOGENERATED - No need to change this section # +resource "incus_instance" "build_container" { + count = var.build_container ? 1 : 0 + + remote = var.incus_remote + project = incus_project.this.name + + name = "build-container" + + image = "images:ubuntu/24.04" + profiles = ["default"] + + config = { + "environment.http_proxy" = var.deploy == "production" ? "http://proxy.ctf-int.internal.nsec.io:3128" : null + "environment.https_proxy" = var.deploy == "production" ? "http://proxy.ctf-int.internal.nsec.io:3128" : null + } + device { + name = "eth0" + type = "nic" + + properties = { + "network" = incus_network.this.name + "name" = "eth0" + } + } + + device { + name = "root" + type = "disk" + + properties = { + "pool" = "default" + "path" = "/" + # !!!! Change this limit if needed !!!! # + "size" = "3GiB" + } + } + + lifecycle { + ignore_changes = [running] + } +} +{% endif %} # AUTOGENERATED - No need to change this section # resource "incus_network_zone_record" "this" { remote = var.incus_remote diff --git a/ctf/templates/new/python-service/deploy.yaml.j2 b/ctf/templates/new/python-service/deploy.yaml.j2 index e57e8a4..081f14d 100644 --- a/ctf/templates/new/python-service/deploy.yaml.j2 +++ b/ctf/templates/new/python-service/deploy.yaml.j2 @@ -2,7 +2,7 @@ # Example on how to run stuff on all hosts of the track - name: "Install Python, PIP and VirtualEnvironment on each host" - hosts: "*" + hosts: all{% if data.with_build %},!build{% endif %} vars_files: - ../track.yaml tasks: @@ -16,7 +16,7 @@ value: "{{ '{{ item.flag }}' }}" ansible.builtin.set_fact: track_flags: "{{ '{{ track_flags | default({}) | combine({key: value}) }}' }}" - + - name: Initial System Upgrade ansible.builtin.apt: update_cache: true @@ -106,10 +106,13 @@ state: restarted enabled: true daemon_reload: true - -# If you have many servers in your track with different deployments, it's probably better to separate them in ansible playbooks and import them like this. -# - import_playbook: main-website.yaml -# - import_playbook: challenge-robots.yaml -# - import_playbook: challenge-lfi.yaml -# - import_playbook: challenge-xxe.yaml - \ No newline at end of file +{% if data.with_build %} + # When using a build container, the unarchive module can be used to install the content on the remote. + - name: Unarchive the content of the build + ansible.builtin.unarchive: + src: /tmp/build.tar + dest: /tmp/ + owner: root + group: root + mode: '0755' +{% endif %} diff --git a/ctf/templates/new/rust-webservice/deploy.yaml.j2 b/ctf/templates/new/rust-webservice/deploy.yaml.j2 index f931866..678feb9 100644 --- a/ctf/templates/new/rust-webservice/deploy.yaml.j2 +++ b/ctf/templates/new/rust-webservice/deploy.yaml.j2 @@ -2,13 +2,13 @@ # Example on how to run stuff on all hosts of the track - name: "Install rust and npm" - hosts: "*" + hosts: all{% if data.with_build %},!build{% endif %} vars_files: - ../track.yaml tasks: # This is a helper task that loads the tracks' `track.yaml` file and loads the flags as # ansible facts (like variables) to use in subsequent steps. The key is the `discourse` tag - # of the flag. See the index.php file for an example on how to use/print the flags. + # of the flag. - name: "Load flags" loop: "{{ '{{ flags }}' }}" vars: @@ -129,9 +129,13 @@ state: restarted enabled: true daemon_reload: true - -# If you have many servers in your track with different deployments, it's probably better to separate them in ansible playbooks and import them like this. -# - import_playbook: main-website.yaml -# - import_playbook: challenge-robots.yaml -# - import_playbook: challenge-lfi.yaml -# - import_playbook: challenge-xxe.yaml +{% if data.with_build %} + # When using a build container, the unarchive module can be used to install the content on the remote. + - name: Unarchive the content of the build + ansible.builtin.unarchive: + src: /tmp/build.tar + dest: /tmp/ + owner: root + group: root + mode: '0755' +{% endif %} diff --git a/ctf/utils.py b/ctf/utils.py index 6d1404c..8fd6bb5 100644 --- a/ctf/utils.py +++ b/ctf/utils.py @@ -10,6 +10,7 @@ from ctf import ENV from ctf.logger import LOG +from ctf.models import Track __CTF_ROOT_DIRECTORY = "" @@ -30,7 +31,7 @@ def check_git_lfs() -> bool: return not bool(subprocess.run(args=["git", "lfs"], capture_output=True).returncode) -def get_all_available_tracks() -> set[str]: +def get_all_available_tracks() -> set[Track]: tracks = set() for entry in os.listdir( @@ -43,34 +44,56 @@ def get_all_available_tracks() -> set[str]: if not os.path.isdir(s=os.path.join(challenges_directory, entry)): continue - tracks.add(entry) + tracks.add(Track(name=entry)) return tracks -def validate_track_can_be_deployed(track: str) -> bool: +def does_track_require_build_container(track: Track) -> bool: + return os.path.isfile( + build_yaml_file_path := os.path.join( + find_ctf_root_directory(), + "challenges", + track.name, + "ansible", + "build.yaml", + ) + ) and bool(load_yaml_file(build_yaml_file_path)) + + +def validate_track_can_be_deployed(track: Track) -> bool: return ( os.path.exists( path=os.path.join( - find_ctf_root_directory(), "challenges", track, "terraform", "main.tf" + find_ctf_root_directory(), + "challenges", + track.name, + "terraform", + "main.tf", ) ) and os.path.exists( path=os.path.join( - find_ctf_root_directory(), "challenges", track, "ansible", "deploy.yaml" + find_ctf_root_directory(), + "challenges", + track.name, + "ansible", + "deploy.yaml", ) ) and os.path.exists( path=os.path.join( - find_ctf_root_directory(), "challenges", track, "ansible", "inventory" + find_ctf_root_directory(), + "challenges", + track.name, + "ansible", + "inventory", ) ) ) -def add_tracks_to_terraform_modules( - tracks: set[str], remote: str, production: bool = False -): +def add_tracks_to_terraform_modules(tracks: set[Track]): with open( file=os.path.join(find_ctf_root_directory(), ".deploy", "modules.tf"), mode="a" ) as fd: @@ -78,15 +101,11 @@ def add_tracks_to_terraform_modules( source=textwrap.dedent( text="""\ {% for track in tracks %} - module "track-{{ track }}" { - source = "../challenges/{{ track }}/terraform" - {% if production %} - deploy = "production" - {% endif %} - {% if remote %} - incus_remote = "{{ remote }}" - {% endif %} - + module "track-{{ track.name }}" { + source = "../challenges/{{ track.name }}/terraform" + build_container = {{ 'true' if track.require_build_container else 'false' }} + {% if track.production %}deploy = "production"{% endif %} + {% if track.remote %}incus_remote = "{{ track.remote }}"{% endif %} depends_on = [module.common] } {% endfor %} @@ -96,8 +115,6 @@ def add_tracks_to_terraform_modules( fd.write( template.render( tracks=tracks - get_terraform_tracks_from_modules(), - production=production, - remote=remote, ) ) @@ -111,43 +128,76 @@ def create_terraform_modules_file(remote: str, production: bool = False): text="""\ module "common" { source = "./common" - {% if production %} - deploy = "production" - {% endif %} - {% if remote %} - incus_remote = "{{ remote }}" - {% endif %} + {% if production %}deploy = "production"{% endif %} + {% if remote %}incus_remote = "{{ remote }}"{% endif %} } + """ ) ) fd.write(template.render(production=production, remote=remote)) -def get_terraform_tracks_from_modules() -> set[str]: +def get_terraform_tracks_from_modules() -> set[Track]: with open( file=os.path.join(find_ctf_root_directory(), ".deploy", "modules.tf"), mode="r" ) as f: modules_tf = f.read() - return set( - re.findall( - pattern=r"^module \"track-([a-z][a-z0-9\-]{0,61}[a-z0-9])\"", - string=modules_tf, - flags=re.MULTILINE, - ) + module_line_regex = re.compile( + r"^module \"track-([a-z][a-z0-9\-]{0,61}[a-z0-9])\"\s*\{$" ) + production_line_regex = re.compile(r"^deploy\s*=\s*\"production\"$") + remote_line_regex = re.compile(r"^incus_remote\s*=\s*\"([^\"]+)\"$") + build_container_line_regex = re.compile(r"^build_container\s*=\s*true$") + + tracks: set[Track] = set() + name: str = "" + remote: str = "local" + production: bool = False + require_build_container: bool = False + + for line in modules_tf.splitlines(): + if not (line := line.strip()): + continue + + if "}" == line and name: + tracks.add( + Track( + name=name, + remote=remote, + production=production, + require_build_container=require_build_container, + ) + ) + name = "" + remote = "local" + production = False + require_build_container = False + continue + + if m := module_line_regex.match(line): + name = m.group(1) + + if production_line_regex.match(line): + production = True + + if m := remote_line_regex.match(line): + remote = m.group(1) + + if build_container_line_regex.match(line): + require_build_container = True + + return tracks def remove_tracks_from_terraform_modules( - tracks: set[str], remote: str, production: bool = False + tracks: set[Track], remote: str, production: bool = False ): current_tracks = get_terraform_tracks_from_modules() create_terraform_modules_file(remote=remote, production=production) - add_tracks_to_terraform_modules( - tracks=(current_tracks - tracks), remote=remote, production=production - ) + add_tracks_to_terraform_modules(tracks=(current_tracks - tracks)) def get_all_file_paths_recursively(path: str) -> Generator[str, None, None]: @@ -175,19 +225,16 @@ def remove_ctf_script_root_directory_from_path(path: str) -> str: return os.path.relpath(path=path, start=find_ctf_root_directory()) +def load_yaml_file(file: str) -> dict[str, Any]: + return yaml.safe_load(stream=open(file, mode="r", encoding="utf-8")) + + def parse_track_yaml(track_name: str) -> dict[str, Any]: - r = yaml.safe_load( - stream=open( - file=( - p := os.path.join( - find_ctf_root_directory(), "challenges", track_name, "track.yaml" - ) - ), - mode="r", - encoding="utf-8", + r = load_yaml_file( + p := os.path.join( + find_ctf_root_directory(), "challenges", track_name, "track.yaml" ) ) - r["file_location"] = remove_ctf_script_root_directory_from_path(path=p) return r @@ -203,14 +250,11 @@ def parse_post_yamls(track_name: str) -> list[dict]: ) ): if post.endswith(".yml") or post.endswith(".yaml"): - with open( - file=os.path.join(posts_dir, post), mode="r", encoding="utf-8" - ) as f: - r = post_data = yaml.safe_load(stream=f) - r["file_location"] = remove_ctf_script_root_directory_from_path( - path=posts_dir - ) - posts.append(post_data) + r = load_yaml_file(os.path.join(posts_dir, post)) + r["file_location"] = remove_ctf_script_root_directory_from_path( + path=posts_dir + ) + posts.append(r) return posts @@ -220,8 +264,8 @@ def find_ctf_root_directory() -> str: if __CTF_ROOT_DIRECTORY: return __CTF_ROOT_DIRECTORY - path = ( - ENV.get("CTF_ROOT_DIR") + path: str = ( + str(ENV.get("CTF_ROOT_DIR")) if "CTF_ROOT_DIR" in ENV else os.path.join(os.getcwd(), ".") ) @@ -236,7 +280,6 @@ def find_ctf_root_directory() -> str: LOG.critical( msg='Could not automatically find the root directory nor the "CTF_ROOT_DIR" environment variable. To initialize a new root directory, use `ctf init [path]`' ) - raise exit(1) LOG.debug(msg=f"Found root directory: {path}") diff --git a/ctf/validators.py b/ctf/validators.py index c4abd2d..4a696e6 100644 --- a/ctf/validators.py +++ b/ctf/validators.py @@ -2,8 +2,8 @@ import glob import os import re -from dataclasses import dataclass +from ctf.models import ValidationError from ctf.utils import ( find_ctf_root_directory, get_all_file_paths_recursively, @@ -13,14 +13,6 @@ ) -@dataclass -class ValidationError: - error_name: str - error_description: str - details: dict[str, str] - track_name: str = "" - - class Validator(abc.ABC): @abc.abstractmethod def validate(self, track_name: str) -> list[ValidationError]: diff --git a/pyproject.toml b/pyproject.toml index 506f628..1c37af0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,9 @@ dependencies = [ "ruff==0.12.8", "setuptools", "typer==0.16.0", + "pydantic" ] -version = "3.0.0" +version = "3.1.0" classifiers = [ "Programming Language :: Python :: 3", "Operating System :: OS Independent",