From 1e9cd03d7995918a090f1b2d1347b8689e338202 Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sun, 13 Oct 2024 13:51:22 -0700 Subject: [PATCH 01/10] allow challenge config to omit provide or pod fields Signed-off-by: Robert Detjens --- src/configparser/challenge.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/configparser/challenge.rs b/src/configparser/challenge.rs index d3ce291..f259f53 100644 --- a/src/configparser/challenge.rs +++ b/src/configparser/challenge.rs @@ -50,13 +50,19 @@ pub fn parse_one(path: &str) -> Result { struct ChallengeConfig { name: String, author: String, + #[serde(default)] category: String, + description: String, difficulty: i64, flag: FlagType, - provide: Vec, - pods: Vec, + + #[serde(default)] + provide: Vec, // optional if no files provided + + #[serde(default)] + pods: Vec, // optional if no containers used } #[derive(Debug, PartialEq, Serialize, Deserialize)] From bb7727b2388b640cdcbb98837737e8225f20d5e1 Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sun, 13 Oct 2024 15:00:52 -0700 Subject: [PATCH 02/10] add example pwn challenge from old test repo Signed-off-by: Robert Detjens --- tests/repo/pwn/notsh/.gitignore | 1 + tests/repo/pwn/notsh/Dockerfile | 42 +++++++++++++++++++ tests/repo/pwn/notsh/Makefile | 23 ++++++++++ tests/repo/pwn/notsh/build-artifacts | 5 +++ tests/repo/pwn/notsh/challenge.yaml | 21 ++++++++++ .../repo/pwn/notsh/container_src/banner_fail | 1 + .../repo/pwn/notsh/container_src/run_chal.sh | 14 +++++++ .../repo/pwn/notsh/container_src/xinetd.conf | 19 +++++++++ tests/repo/pwn/notsh/flag | 1 + tests/repo/pwn/notsh/src/bestpwn.c | 32 ++++++++++++++ tests/repo/web/bar/Containerfile | 2 +- 11 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/repo/pwn/notsh/.gitignore create mode 100644 tests/repo/pwn/notsh/Dockerfile create mode 100644 tests/repo/pwn/notsh/Makefile create mode 100644 tests/repo/pwn/notsh/build-artifacts create mode 100644 tests/repo/pwn/notsh/challenge.yaml create mode 100644 tests/repo/pwn/notsh/container_src/banner_fail create mode 100644 tests/repo/pwn/notsh/container_src/run_chal.sh create mode 100644 tests/repo/pwn/notsh/container_src/xinetd.conf create mode 100644 tests/repo/pwn/notsh/flag create mode 100644 tests/repo/pwn/notsh/src/bestpwn.c diff --git a/tests/repo/pwn/notsh/.gitignore b/tests/repo/pwn/notsh/.gitignore new file mode 100644 index 0000000..5761abc --- /dev/null +++ b/tests/repo/pwn/notsh/.gitignore @@ -0,0 +1 @@ +*.o diff --git a/tests/repo/pwn/notsh/Dockerfile b/tests/repo/pwn/notsh/Dockerfile new file mode 100644 index 0000000..d8bdd5a --- /dev/null +++ b/tests/repo/pwn/notsh/Dockerfile @@ -0,0 +1,42 @@ +# IMAGE 1: build challenge +# @AUTHOR: if your chal doesn't build seperately from being run (i.e. Python), +# delete all of the IMAGE 1 code +FROM ubuntu:18.04 AS builder + +# @AUTHOR: build requirements here +RUN apt-get -qq update && apt-get -qq --no-install-recommends install build-essential + +WORKDIR /build + +# @AUTHOR: make sure all source is copied in. If everything is in src/, no change needed +COPY src ./src/ +COPY Makefile . +RUN make container + +# IMAGE 2: run challenge +# @AUTHOR: feel free to change base image as necessary (i.e. python, node) +FROM ubuntu:18.04 + +# @AUTHOR: run requirements here +RUN apt-get -qq update && apt-get -qq --no-install-recommends install xinetd + +# copy binary +WORKDIR /chal +# @AUTHOR: make sure all build outputs are copied to the runner +# if there is no build output, replace this with the appropriate COPY stmts +# to pull files from the host +COPY --from=builder /build/notsh /chal/ + +# copy flag +COPY flag /chal/ + +# make user +RUN useradd chal + +# copy service info +COPY container_src/* / + +# run challenge +EXPOSE 31337 +RUN chmod +x /run_chal.sh +CMD ["/usr/sbin/xinetd", "-syslog", "local0", "-dontfork", "-f", "/xinetd.conf"] diff --git a/tests/repo/pwn/notsh/Makefile b/tests/repo/pwn/notsh/Makefile new file mode 100644 index 0000000..bdcdd8d --- /dev/null +++ b/tests/repo/pwn/notsh/Makefile @@ -0,0 +1,23 @@ +CC=gcc +C_FLAGS=-Wall # disable NX: -z execstack + # disable canary: -fno-stack-protector + # disable PIE: -no-pie +C_LIBS= # -lcrypto or something + +out=notsh + +.PHONY: all +all: $(out) + +$(out): src/*.c + $(CC) $(C_FLAGS) -o $@ $^ $(C_LIBS) + +# container builds this target +# make sure 'all' builds everything you need +# container builds on ubu1804 +.PHONY: container +container: all + +.PHONY: clean +clean: + $(RM) $(out) *.o diff --git a/tests/repo/pwn/notsh/build-artifacts b/tests/repo/pwn/notsh/build-artifacts new file mode 100644 index 0000000..b8e3f4e --- /dev/null +++ b/tests/repo/pwn/notsh/build-artifacts @@ -0,0 +1,5 @@ +# These need to be absolute paths +/chal/notsh + +# libc path on Ubuntu +/lib/x86_64-linux-gnu/libc.so.6 diff --git a/tests/repo/pwn/notsh/challenge.yaml b/tests/repo/pwn/notsh/challenge.yaml new file mode 100644 index 0000000..1cf42ce --- /dev/null +++ b/tests/repo/pwn/notsh/challenge.yaml @@ -0,0 +1,21 @@ +name: notsh +author: captainGeech +description: |- + This challenge isn't a shell + + `nc {{host}} {{port}}` + +provide: +- ./notsh.zip + +flag: + file: ./flag + +pods: + - name: main + build: . + replicas: 2 + ports: + - internal: 31337 + expose: + tcp: 30124 diff --git a/tests/repo/pwn/notsh/container_src/banner_fail b/tests/repo/pwn/notsh/container_src/banner_fail new file mode 100644 index 0000000..2cfe885 --- /dev/null +++ b/tests/repo/pwn/notsh/container_src/banner_fail @@ -0,0 +1 @@ +XINETD CONNECTION FAILED, PING @ADMIN \ No newline at end of file diff --git a/tests/repo/pwn/notsh/container_src/run_chal.sh b/tests/repo/pwn/notsh/container_src/run_chal.sh new file mode 100644 index 0000000..9baef62 --- /dev/null +++ b/tests/repo/pwn/notsh/container_src/run_chal.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# no stderr +exec 2>/dev/null + +# dir +cd /chal + +# timeout after 20 sec +# @AUTHOR: make sure to set the propery entry point +# <---| don't touch anything left +# | unless you need a longer timeout +timeout -k1 20 stdbuf -i0 -o0 -e0 ./notsh +# ^^ 20 sec timeout \ No newline at end of file diff --git a/tests/repo/pwn/notsh/container_src/xinetd.conf b/tests/repo/pwn/notsh/container_src/xinetd.conf new file mode 100644 index 0000000..fe08d56 --- /dev/null +++ b/tests/repo/pwn/notsh/container_src/xinetd.conf @@ -0,0 +1,19 @@ +service chal +{ + socket_type = stream + protocol = tcp + wait = no + user = chal + type = UNLISTED + bind = 0.0.0.0 + port = 31337 + server = /run_chal.sh + banner_fail = /banner_fail + + # these may need to be adjusted based on how resource + # intensive the challenge is (along with k8s scaling) + nice = 2 + rlimit_cpu = 10 + cps = 10000 10 + instances = 10 +} diff --git a/tests/repo/pwn/notsh/flag b/tests/repo/pwn/notsh/flag new file mode 100644 index 0000000..4fa6e91 --- /dev/null +++ b/tests/repo/pwn/notsh/flag @@ -0,0 +1 @@ +dam{good_test_chal_notsh} diff --git a/tests/repo/pwn/notsh/src/bestpwn.c b/tests/repo/pwn/notsh/src/bestpwn.c new file mode 100644 index 0000000..014009e --- /dev/null +++ b/tests/repo/pwn/notsh/src/bestpwn.c @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include +#include + +int main() { + char input[20] = {0}; + char flag[40] = {0}; + + puts("hello from notsh v1.0"); + printf("would you like a flag? "); + + fgets(input, 20, stdin); + + input[strcspn(input, "\n")] = 0; + + if (strcmp(input, "yes") == 0) { + puts("ok!"); + + int fd = open("./flag", O_RDONLY); + read(fd, flag, 40); + write(1, flag, 40); + } else if (strcmp(input, "shell") == 0) { + system("/bin/sh"); + } else { + puts("better luck next time!"); + } + + return 0; +} \ No newline at end of file diff --git a/tests/repo/web/bar/Containerfile b/tests/repo/web/bar/Containerfile index 6d7b659..0c82457 100644 --- a/tests/repo/web/bar/Containerfile +++ b/tests/repo/web/bar/Containerfile @@ -1,3 +1,3 @@ FROM nginx -COPY site_source/ /var/www/html/ +COPY site_source/ /usr/share/nginx/html/ From a9dd60a1144812a0f59e96d3715d43ba2170f529 Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sun, 13 Oct 2024 15:53:15 -0700 Subject: [PATCH 03/10] challenge image/build parsing improvements - coerce string `build:` directory into full build context object - allow only one of either `build:` or `image:` fields Signed-off-by: Robert Detjens --- Cargo.lock | 7 ++++ Cargo.toml | 1 + src/configparser/challenge.rs | 43 +++++++++++++++++------ src/configparser/field_coersion.rs | 55 ++++++++++++++++++++++++++++++ src/configparser/mod.rs | 1 + 5 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 src/configparser/field_coersion.rs diff --git a/Cargo.lock b/Cargo.lock index de9c19c..a0c7999 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,6 +223,7 @@ dependencies = [ "simplelog", "tera", "tokio", + "void", ] [[package]] @@ -2421,6 +2422,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 31f0c45..b0cadd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ serde_yaml = "0.9" tera = "1.19.1" simplelog = { version = "0.12.2", features = ["paris"] } fully_pub = "0.1.4" +void = "1" # kubernetes: kube = { version = "0.91.0", features = ["runtime", "derive"] } diff --git a/src/configparser/challenge.rs b/src/configparser/challenge.rs index f259f53..ccc58ec 100644 --- a/src/configparser/challenge.rs +++ b/src/configparser/challenge.rs @@ -6,8 +6,11 @@ use simplelog::*; use std::collections::BTreeMap; use std::fs; use std::path::Path; +use std::str::FromStr; +use void::Void; use crate::configparser::config::Resource; +use crate::configparser::field_coersion::string_or_struct; pub fn parse_all() -> Vec> { // find all challenge.yaml files @@ -50,12 +53,14 @@ pub fn parse_one(path: &str) -> Result { struct ChallengeConfig { name: String, author: String, + description: String, #[serde(default)] category: String, - description: String, + #[serde(default = "default_difficulty")] difficulty: i64, + flag: FlagType, #[serde(default)] @@ -65,6 +70,10 @@ struct ChallengeConfig { pods: Vec, // optional if no containers used } +fn default_difficulty() -> i64 { + 1 +} + #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] #[fully_pub] @@ -104,8 +113,10 @@ struct FileVerifier { #[fully_pub] struct Pod { name: String, - build: BuildSpec, - image: String, + + #[serde(flatten)] + image_source: ImageSource, + env: Option, resources: Option, replicas: i64, @@ -114,20 +125,32 @@ struct Pod { } #[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] +#[serde(rename_all = "lowercase")] #[fully_pub] -enum BuildSpec { - Context(String), - Map(BTreeMap), +enum ImageSource { + #[serde(deserialize_with = "string_or_struct")] + Build(BuildObject), + Image(String), } #[derive(Debug, PartialEq, Serialize, Deserialize)] #[fully_pub] struct BuildObject { context: String, - dockerfile: String, - dockerfile_inline: String, - args: ListOrMap, + dockerfile: Option, + // dockerfile_inline: String, + #[serde(default)] + args: BTreeMap, +} +impl FromStr for BuildObject { + type Err = Void; + fn from_str(s: &str) -> std::result::Result { + Ok(BuildObject { + context: s.to_string(), + dockerfile: None, + args: Default::default(), + }) + } } #[derive(Debug, PartialEq, Serialize, Deserialize)] diff --git a/src/configparser/field_coersion.rs b/src/configparser/field_coersion.rs new file mode 100644 index 0000000..2c51e33 --- /dev/null +++ b/src/configparser/field_coersion.rs @@ -0,0 +1,55 @@ +// stuff to coerce bare string into full build context object +// (based on serde example: https://serde.rs/string-or-struct.html) + +use std::collections::BTreeMap as Map; +use std::fmt; +use std::marker::PhantomData; +use std::str::FromStr; + +use serde::de::{self, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use void::Void; + +pub fn string_or_struct<'de, T, D>(deserializer: D) -> Result +where + T: Deserialize<'de> + FromStr, + D: Deserializer<'de>, +{ + // This is a Visitor that forwards string types to T's `FromStr` impl and + // forwards map types to T's `Deserialize` impl. The `PhantomData` is to + // keep the compiler from complaining about T being an unused generic type + // parameter. We need T in order to know the Value type for the Visitor + // impl. + struct StringOrStruct(PhantomData T>); + + impl<'de, T> Visitor<'de> for StringOrStruct + where + T: Deserialize<'de> + FromStr, + { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("string or map") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(FromStr::from_str(value).unwrap()) + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + // `MapAccessDeserializer` is a wrapper that turns a `MapAccess` + // into a `Deserializer`, allowing it to be used as the input to T's + // `Deserialize` implementation. T then deserializes itself using + // the entries from the map visitor. + Deserialize::deserialize(de::value::MapAccessDeserializer::new(map)) + } + } + + deserializer.deserialize_any(StringOrStruct(PhantomData)) +} diff --git a/src/configparser/mod.rs b/src/configparser/mod.rs index 79ab3ce..eb5aa45 100644 --- a/src/configparser/mod.rs +++ b/src/configparser/mod.rs @@ -1,5 +1,6 @@ pub mod challenge; pub mod config; +pub mod field_coersion; use anyhow::{anyhow, Error, Result}; pub use config::UserPass; // reexport From df3fd305002701cb064fbe2fc572a5cfc11fc21b Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sun, 13 Oct 2024 18:20:14 -0700 Subject: [PATCH 04/10] validate challenges listed in profile deploy settings exist also adds a directory field to ChallengeConfig struct to hold the reference path for the challenge Signed-off-by: Robert Detjens --- src/commands/validate.rs | 49 ++++++++++++++++++++++++++++++----- src/configparser/challenge.rs | 19 +++++++++++--- src/configparser/config.rs | 12 ++++++++- src/configparser/mod.rs | 7 +++++ tests/repo/rcds.yaml | 5 ++-- 5 files changed, 79 insertions(+), 13 deletions(-) diff --git a/src/commands/validate.rs b/src/commands/validate.rs index 795c575..07167e0 100644 --- a/src/commands/validate.rs +++ b/src/commands/validate.rs @@ -1,25 +1,62 @@ -use crate::configparser::{get_challenges, get_config}; use simplelog::*; +use std::path::Path; use std::process::exit; +use crate::configparser::{get_challenges, get_config, get_profile_deploy}; + pub fn run() { info!("validating config..."); - match get_config() { - Ok(_) => info!(" config ok!"), + let config = match get_config() { + Ok(c) => c, Err(err) => { error!("{err:#}"); exit(1); } - } + }; + info!(" config ok!"); info!("validating challenges..."); - match get_challenges() { - Ok(_) => info!(" challenges ok!"), + let chals = match get_challenges() { + Ok(c) => c, Err(errors) => { for e in errors.iter() { error!("{e:#}"); } exit(1); } + }; + info!(" challenges ok!"); + + // check global deploy settings for invalid challenges + info!("validating deploy config..."); + for (profile_name, _pconfig) in config.profiles.iter() { + // get em + let deploy_challenges = match get_profile_deploy(profile_name) { + Ok(d) => &d.challenges, + Err(err) => { + error!("{err:#}"); + exit(1); + } + }; + + // check em + let missing: Vec<_> = deploy_challenges + .keys() + .filter_map( + // invert match to filter for challenges that *dont* match + |path| match chals.iter().find(|c| c.directory == Path::new(path)) { + Some(_) => None, + None => Some(path), + }, + ) + .collect(); + if missing.len() > 0 { + error!( + "Deploy settings for profile '{profile_name}' has challenges that do not exist:" + ); + missing.iter().for_each(|path| error!(" - {path}")); + exit(1) + } } + info!(" deploy ok!") } diff --git a/src/configparser/challenge.rs b/src/configparser/challenge.rs index ccc58ec..2bd05a2 100644 --- a/src/configparser/challenge.rs +++ b/src/configparser/challenge.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use simplelog::*; use std::collections::BTreeMap; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use void::Void; @@ -26,21 +26,29 @@ pub fn parse_all() -> Vec> { } pub fn parse_one(path: &str) -> Result { - trace!("trying to parse {path}"); + debug!("trying to parse {path}"); // extract category from challenge path let contents = fs::read_to_string(path)?; let mut parsed: ChallengeConfig = serde_yaml::from_str(&contents)?; - let category = Path::new(path) + // safe to unwrap here since path from find() always has the challenge yaml + let pathobj = Path::new(path).parent().unwrap(); + parsed.directory = pathobj.strip_prefix("./").unwrap_or(pathobj).to_path_buf(); + + let category = parsed + .directory .components() - .nth_back(2) + .nth_back(1) .expect("could not find category from path"); category .as_os_str() .to_str() .unwrap() .clone_into(&mut parsed.category); + + trace!("got chal: {parsed:#?}"); + Ok(parsed) } @@ -55,6 +63,9 @@ struct ChallengeConfig { author: String, description: String, + #[serde(default)] + directory: PathBuf, + #[serde(default)] category: String, diff --git a/src/configparser/config.rs b/src/configparser/config.rs index 73deffb..ccbd612 100644 --- a/src/configparser/config.rs +++ b/src/configparser/config.rs @@ -6,11 +6,13 @@ use std::collections::BTreeMap; use std::fs; pub fn parse() -> Result { - trace!("trying to parse rcds.yaml"); + debug!("trying to parse rcds.yaml"); let contents = fs::read_to_string("rcds.yaml").with_context(|| "failed to read rcds.yaml")?; let parsed = serde_yaml::from_str(&contents).with_context(|| "failed to parse rcds.yaml")?; + trace!("got config: {parsed:#?}"); + Ok(parsed) } @@ -24,6 +26,7 @@ struct RcdsConfig { flag_regex: String, registry: Registry, defaults: Defaults, + deploy: BTreeMap, profiles: BTreeMap, points: Vec, } @@ -57,6 +60,13 @@ struct Defaults { resources: Resource, } +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[fully_pub] +struct ProfileDeploy { + #[serde(flatten)] + challenges: BTreeMap, +} + #[derive(Debug, PartialEq, Serialize, Deserialize)] #[fully_pub] struct ProfileConfig { diff --git a/src/configparser/mod.rs b/src/configparser/mod.rs index eb5aa45..8f940dc 100644 --- a/src/configparser/mod.rs +++ b/src/configparser/mod.rs @@ -36,6 +36,13 @@ pub fn get_profile_config(profile_name: &str) -> Result<&config::ProfileConfig> .get(profile_name) .ok_or(anyhow!("profile {profile_name} not found in config")) } +/// Get challenge deploy config struct for the passed profile name +pub fn get_profile_deploy(profile_name: &str) -> Result<&config::ProfileDeploy> { + get_config()? + .deploy + .get(profile_name) + .ok_or(anyhow!("profile {profile_name} not found in deploy config")) +} /// get challenges from global, or load from files if not parsed yet pub fn get_challenges() -> Result> { diff --git a/tests/repo/rcds.yaml b/tests/repo/rcds.yaml index 7ca7c14..99f23f4 100644 --- a/tests/repo/rcds.yaml +++ b/tests/repo/rcds.yaml @@ -22,8 +22,9 @@ points: deploy: # control challenge deployment status explicitly per environment/profile testing: - misc/foo: true - rev/bar: false + misc/garf: true + pwn/notsh: true + web/bar: false profiles: # configure per-environment credentials etc From 5f4595b743af166a20c6c85a8b3e3e33af251a8e Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sun, 13 Oct 2024 22:43:59 -0700 Subject: [PATCH 05/10] use HashMap instead of BTreeMap for config structs bollard wants to use HashMap to talk to the docker daemon, and we don't need the extra features BTreeMap offers. simpler to agree on HashMap. Signed-off-by: Robert Detjens --- src/configparser/challenge.rs | 16 ++++++++++------ src/configparser/config.rs | 10 +++++----- src/configparser/field_coersion.rs | 2 +- src/configparser/mod.rs | 1 + 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/configparser/challenge.rs b/src/configparser/challenge.rs index 2bd05a2..86e6eae 100644 --- a/src/configparser/challenge.rs +++ b/src/configparser/challenge.rs @@ -3,7 +3,7 @@ use fully_pub::fully_pub; use rust_search::SearchBuilder; use serde::{Deserialize, Serialize}; use simplelog::*; -use std::collections::BTreeMap; +use std::collections::HashMap as Map; use std::fs; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -58,7 +58,7 @@ pub fn parse_one(path: &str) -> Result { #[derive(Debug, PartialEq, Serialize, Deserialize)] #[fully_pub] -struct ChallengeConfig { +pub struct ChallengeConfig { name: String, author: String, description: String, @@ -148,28 +148,32 @@ enum ImageSource { #[fully_pub] struct BuildObject { context: String, - dockerfile: Option, + #[serde(default = "default_dockerfile")] + dockerfile: String, // dockerfile_inline: String, #[serde(default)] - args: BTreeMap, + args: Map, } impl FromStr for BuildObject { type Err = Void; fn from_str(s: &str) -> std::result::Result { Ok(BuildObject { context: s.to_string(), - dockerfile: None, + dockerfile: default_dockerfile(), args: Default::default(), }) } } +fn default_dockerfile() -> String { + "Dockerfile".to_string() +} #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] #[fully_pub] enum ListOrMap { List(Vec), - Map(BTreeMap), + Map(Map), } #[derive(Debug, PartialEq, Serialize, Deserialize)] diff --git a/src/configparser/config.rs b/src/configparser/config.rs index ccbd612..85dac58 100644 --- a/src/configparser/config.rs +++ b/src/configparser/config.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use fully_pub::fully_pub; use serde::{Deserialize, Serialize}; use simplelog::*; -use std::collections::BTreeMap; +use std::collections::HashMap as Map; use std::fs; pub fn parse() -> Result { @@ -26,8 +26,8 @@ struct RcdsConfig { flag_regex: String, registry: Registry, defaults: Defaults, - deploy: BTreeMap, - profiles: BTreeMap, + deploy: Map, + profiles: Map, points: Vec, } @@ -64,13 +64,13 @@ struct Defaults { #[fully_pub] struct ProfileDeploy { #[serde(flatten)] - challenges: BTreeMap, + challenges: Map, } #[derive(Debug, PartialEq, Serialize, Deserialize)] #[fully_pub] struct ProfileConfig { - // deployed_challenges: BTreeMap, + // deployed_challenges: HashMap, frontend_url: String, frontend_token: Option, challenges_domain: String, diff --git a/src/configparser/field_coersion.rs b/src/configparser/field_coersion.rs index 2c51e33..51b8182 100644 --- a/src/configparser/field_coersion.rs +++ b/src/configparser/field_coersion.rs @@ -1,7 +1,7 @@ // stuff to coerce bare string into full build context object // (based on serde example: https://serde.rs/string-or-struct.html) -use std::collections::BTreeMap as Map; +use std::collections::HashMap as Map; use std::fmt; use std::marker::PhantomData; use std::str::FromStr; diff --git a/src/configparser/mod.rs b/src/configparser/mod.rs index 8f940dc..aca5ba8 100644 --- a/src/configparser/mod.rs +++ b/src/configparser/mod.rs @@ -3,6 +3,7 @@ pub mod config; pub mod field_coersion; use anyhow::{anyhow, Error, Result}; +pub use challenge::ChallengeConfig; // reexport pub use config::UserPass; // reexport use itertools::Itertools; use simplelog::*; From 83979c66e1235aaaf494b8944b6587fcf1d195bc Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sun, 13 Oct 2024 22:53:08 -0700 Subject: [PATCH 06/10] initial image building code (build only) does not push yet, and probably missing other stuff Signed-off-by: Robert Detjens --- Cargo.lock | 174 +++++++++++++++++++++++------- Cargo.toml | 6 +- src/access_handlers/docker.rs | 9 +- src/builder/docker.rs | 84 +++++++++++++++ src/builder/mod.rs | 80 ++++++++++++++ src/commands/build.rs | 12 ++- src/lib.rs | 1 + tests/repo/rcds.yaml | 2 +- tests/repo/web/bar/challenge.yaml | 4 +- 9 files changed, 323 insertions(+), 49 deletions(-) create mode 100644 src/builder/docker.rs diff --git a/Cargo.lock b/Cargo.lock index a0c7999..eb99096 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,6 +221,8 @@ dependencies = [ "serde", "serde_yaml", "simplelog", + "tar", + "tempfile", "tera", "tokio", "void", @@ -335,7 +337,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -597,6 +599,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "event-listener" version = "5.3.1" @@ -618,6 +630,24 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -865,9 +895,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -1235,9 +1265,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.154" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libm" @@ -1255,6 +1285,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "lock_api" version = "0.4.12" @@ -1397,9 +1433,9 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -1602,6 +1638,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.1" @@ -1685,6 +1730,19 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustix" +version = "0.38.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustls" version = "0.23.9" @@ -2051,6 +2109,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tar" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "tera" version = "1.19.1" @@ -2191,9 +2273,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -2544,7 +2626,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2562,7 +2644,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -2582,18 +2673,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -2604,9 +2695,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -2616,9 +2707,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -2628,15 +2719,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -2646,9 +2737,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -2658,9 +2749,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -2670,9 +2761,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -2682,9 +2773,20 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "xattr" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] [[package]] name = "zerocopy" diff --git a/Cargo.toml b/Cargo.toml index b0cadd8..00fde1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,14 @@ tera = "1.19.1" simplelog = { version = "0.12.2", features = ["paris"] } fully_pub = "0.1.4" void = "1" +futures-util = "0.3.30" # kubernetes: kube = { version = "0.91.0", features = ["runtime", "derive"] } k8s-openapi = { version = "0.22.0", features = ["latest"] } tokio = { version = "1.38.0", features = ["rt", "macros"] } + +# docker: bollard = "0.16.1" -futures-util = "0.3.30" +tar = "0.4.42" +tempfile = "3.13.0" diff --git a/src/access_handlers/docker.rs b/src/access_handlers/docker.rs index 8bb4c72..94b9489 100644 --- a/src/access_handlers/docker.rs +++ b/src/access_handlers/docker.rs @@ -9,6 +9,7 @@ use itertools::Itertools; use simplelog::*; use tokio; +use crate::builder::docker::client; use crate::configparser::{get_config, get_profile_config}; /// container registry / daemon access checks @@ -45,14 +46,6 @@ pub async fn check(profile_name: &str) -> Result<()> { Ok(()) } -async fn client() -> Result { - debug!("connecting to docker..."); - let client = Docker::connect_with_defaults()?; - client.ping().await?; - - Ok(client) -} - /// test build-time registry push credentials by pushing test image async fn check_build_credentials(client: &Docker, test_image: &str) -> Result<(), Error> { // do we have push access to registry? diff --git a/src/builder/docker.rs b/src/builder/docker.rs new file mode 100644 index 0000000..3e6f502 --- /dev/null +++ b/src/builder/docker.rs @@ -0,0 +1,84 @@ +use anyhow::{anyhow, Context, Error, Result}; +use bollard::{image::BuildImageOptions, Docker}; +use futures_util::{StreamExt, TryStreamExt}; +use simplelog::*; +use std::{io::Read, path::Path}; +use tar; +use tempfile::tempfile; +use tokio; + +use crate::configparser::challenge::BuildObject; + +#[tokio::main(flavor = "current_thread")] // make this a sync function +pub async fn build_image(context: &Path, options: &BuildObject, tag: &str) -> Result { + trace!("building image in directory {context:?} to tag {tag:?}"); + let client = client() + .await + // truncate error chain with new error (returned error is way too verbose) + .map_err(|_| anyhow!("could not talk to Docker daemon (is DOCKER_HOST correct?)"))?; + + let build_opts = BuildImageOptions { + dockerfile: options.dockerfile.clone(), + buildargs: options.args.clone(), + t: tag.to_string(), + forcerm: true, + ..Default::default() + }; + + // tar up image context + // TODO: dont store the tarball in memory... + // let mut tar = tar::Builder::new(tempfile()?); + let mut tar = tar::Builder::new(Vec::new()); + tar.append_dir_all("", context.join(&options.context)) + .with_context(|| "could not create image context tarball")?; + let tarball = tar.into_inner()?; + + // send to docker daemon + let mut build_stream = client.build_image(build_opts, None, Some(tarball.into())); + + // stream output to stdout + while let Some(msg) = build_stream.next().await { + match msg?.stream { + Some(log) => info!( + "building {}: {}", + context.to_string_lossy(), + log.trim() + ), + None => (), + } + } + + Ok("".to_string()) +} + +// +// helper functions +// +pub async fn client() -> Result { + debug!("connecting to docker..."); + let client = Docker::connect_with_defaults()?; + client.ping().await?; + + Ok(client) +} + +#[derive(Debug)] +pub enum EngineType { + Docker, + Podman, +} +pub async fn engine_type() -> EngineType { + let c = client().await.unwrap(); + let version = c.version().await.unwrap(); + + if version + .components + .unwrap() + .iter() + .any(|c| c.name == "Podman Engine") + { + EngineType::Podman + } else { + EngineType::Docker + } +} diff --git a/src/builder/mod.rs b/src/builder/mod.rs index e69de29..89b8b08 100644 --- a/src/builder/mod.rs +++ b/src/builder/mod.rs @@ -0,0 +1,80 @@ +// the thing that builds the stuff +// what more is there to say + +use anyhow::{anyhow, Error, Result}; +use bollard::image::BuildImageOptions; +use futures_util::stream::Iter; +use itertools::Itertools; +use simplelog::*; +use std::default; +use std::fmt::Pointer; +use std::iter::zip; +use std::path::Path; + +use crate::configparser::challenge::{BuildObject, ChallengeConfig, ImageSource::*}; +use crate::configparser::{get_challenges, get_config, get_profile_config, get_profile_deploy}; + +pub mod docker; +use docker::build_image; + +/// Build all enabled challenges for the given profile +pub fn build_challenges(profile_name: &str) -> Result<()> { + for chal in enabled_challenges(profile_name)? { + build_challenge_images(profile_name, &chal); + } + + Ok(()) +} + +/// Get all enabled challenges for profile +pub fn enabled_challenges(profile_name: &str) -> Result> { + let config = get_config()?; + let challenges = get_challenges().unwrap(); + let deploy = &get_profile_deploy(profile_name)?.challenges; + + let enabled = deploy + .iter() + .filter_map(|(chal, enabled)| match enabled { + true => challenges.iter().find(|c| c.directory == Path::new(chal)), + false => None, + }) + .collect(); + + Ok(enabled) +} + +/// Build all images for challenge under given path, return image tag +fn build_challenge_images(profile_name: &str, chal: &ChallengeConfig) -> String { + debug!("building images for chal {:?}", chal.directory); + let build_infos: Vec<_> = chal + .pods + .iter() + .filter_map(|c| match &c.image_source { + Image(_) => None, + Build(b) => Some(b), + }) + .collect(); + + for (opts, tag) in zip(build_infos, challenge_image_tags(profile_name, chal)) { + docker::build_image(&chal.directory, opts, &tag); + } + + "".to_string() +} + +fn challenge_image_tags(profile_name: &str, chal: &ChallengeConfig) -> Vec { + let config = get_config().unwrap(); + + chal.pods + .iter() + .map(|image| { + format!( + "{registry}/{challenge}-{container}:{profile}", + registry = config.registry.domain, + challenge = chal.name, + container = image.name, + profile = profile_name + ) + }) + .collect() +} diff --git a/src/commands/build.rs b/src/commands/build.rs index 444e369..8bc60c8 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -1,3 +1,11 @@ -pub fn run(_profile: &str, _push: &bool) { - println!("running build!"); +use simplelog::*; +use std::process::exit; + +use crate::builder::build_challenges; +use crate::configparser::{get_config, get_profile_config}; + +pub fn run(profile_name: &str, push: &bool) { + info!("building images..."); + + build_challenges(profile_name); } diff --git a/src/lib.rs b/src/lib.rs index c8b5938..58f3173 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ // we dont need unused variables etc warnings while we're working on it pub mod access_handlers; +pub mod builder; pub mod commands; pub mod configparser; diff --git a/tests/repo/rcds.yaml b/tests/repo/rcds.yaml index 99f23f4..53f823d 100644 --- a/tests/repo/rcds.yaml +++ b/tests/repo/rcds.yaml @@ -24,7 +24,7 @@ deploy: testing: misc/garf: true pwn/notsh: true - web/bar: false + web/bar: true profiles: # configure per-environment credentials etc diff --git a/tests/repo/web/bar/challenge.yaml b/tests/repo/web/bar/challenge.yaml index c940313..05fda66 100644 --- a/tests/repo/web/bar/challenge.yaml +++ b/tests/repo/web/bar/challenge.yaml @@ -13,7 +13,9 @@ flag: # each individual pod is gonna allow only 1 container for now pods: - name: bar - build: ./ + build: + context: . + dockerfile: Containerfile replicas: 1 ports: - internal: 80 From f7cdb9c3c5558d47d2da3d75c144195063581bf6 Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Thu, 17 Oct 2024 22:32:17 -0700 Subject: [PATCH 07/10] Improve error reporting for container build errors Signed-off-by: Robert Detjens --- src/builder/docker.rs | 43 +++++++++++++++++++------- src/builder/mod.rs | 70 +++++++++++++++++++++---------------------- src/commands/build.rs | 9 +++++- tests/repo/rcds.yaml | 2 +- 4 files changed, 77 insertions(+), 47 deletions(-) diff --git a/src/builder/docker.rs b/src/builder/docker.rs index 3e6f502..4c43227 100644 --- a/src/builder/docker.rs +++ b/src/builder/docker.rs @@ -1,5 +1,9 @@ -use anyhow::{anyhow, Context, Error, Result}; -use bollard::{image::BuildImageOptions, Docker}; +use anyhow::{anyhow, bail, Context, Error, Result}; +use bollard::auth::DockerCredentials; +use bollard::errors::Error as DockerError; +use bollard::image::{BuildImageOptions, PushImageOptions}; +use bollard::Docker; +use core::fmt; use futures_util::{StreamExt, TryStreamExt}; use simplelog::*; use std::{io::Read, path::Path}; @@ -8,6 +12,7 @@ use tempfile::tempfile; use tokio; use crate::configparser::challenge::BuildObject; +use crate::configparser::UserPass; #[tokio::main(flavor = "current_thread")] // make this a sync function pub async fn build_image(context: &Path, options: &BuildObject, tag: &str) -> Result { @@ -37,14 +42,32 @@ pub async fn build_image(context: &Path, options: &BuildObject, tag: &str) -> Re let mut build_stream = client.build_image(build_opts, None, Some(tarball.into())); // stream output to stdout - while let Some(msg) = build_stream.next().await { - match msg?.stream { - Some(log) => info!( - "building {}: {}", - context.to_string_lossy(), - log.trim() - ), - None => (), + while let Some(item) = build_stream.next().await { + match item { + // error from stream? + Err(e) => match e { + DockerError::DockerStreamError { error } => bail!("build error: {error}"), + other => bail!("build error: {other:?}"), + }, + Ok(msg) => { + // error from daemon? + if let Some(e) = msg.error_detail { + bail!( + "error building image: {}", + e.message.unwrap_or("".to_string()) + ) + } + + match msg.stream { + Some(log) => info!( + "building {}: {}", + context.to_string_lossy(), + // tag, + log.trim() + ), + None => (), + } + } } } diff --git a/src/builder/mod.rs b/src/builder/mod.rs index 89b8b08..6a6c82b 100644 --- a/src/builder/mod.rs +++ b/src/builder/mod.rs @@ -1,7 +1,7 @@ // the thing that builds the stuff // what more is there to say -use anyhow::{anyhow, Error, Result}; +use anyhow::{anyhow, Context, Error, Result}; use bollard::image::BuildImageOptions; use futures_util::stream::Iter; use itertools::Itertools; @@ -15,15 +15,15 @@ use crate::configparser::challenge::{BuildObject, ChallengeConfig, ImageSource:: use crate::configparser::{get_challenges, get_config, get_profile_config, get_profile_deploy}; pub mod docker; -use docker::build_image; +use docker::{build_image, push_image}; -/// Build all enabled challenges for the given profile -pub fn build_challenges(profile_name: &str) -> Result<()> { - for chal in enabled_challenges(profile_name)? { - build_challenge_images(profile_name, &chal); - } - - Ok(()) +/// Build all enabled challenges for the given profile. Returns tags built +pub fn build_challenges(profile_name: &str) -> Result> { + enabled_challenges(profile_name)? + .iter() + .map(|chal| build_challenge_images(profile_name, chal)) + .flatten_ok() + .collect::>() } /// Get all enabled challenges for profile @@ -44,37 +44,37 @@ pub fn enabled_challenges(profile_name: &str) -> Result> { } /// Build all images for challenge under given path, return image tag -fn build_challenge_images(profile_name: &str, chal: &ChallengeConfig) -> String { +fn build_challenge_images(profile_name: &str, chal: &ChallengeConfig) -> Result> { debug!("building images for chal {:?}", chal.directory); - let build_infos: Vec<_> = chal + let config = get_config()?; + + let built_tags = chal .pods .iter() - .filter_map(|c| match &c.image_source { + .filter_map(|p| match &p.image_source { Image(_) => None, - Build(b) => Some(b), + Build(b) => { + let tag = format!( + "{registry}/{challenge}-{container}:{profile}", + registry = config.registry.domain, + challenge = chal.name, + container = p.name, + profile = profile_name + ); + Some( + docker::build_image(&chal.directory, b, &tag).with_context(|| { + format!( + "error building image {} for chal {}", + p.name, + chal.directory.to_string_lossy() + ) + }), + ) + } }) - .collect(); + .collect::>()?; - for (opts, tag) in zip(build_infos, challenge_image_tags(profile_name, chal)) { - docker::build_image(&chal.directory, opts, &tag); - } + trace!("built these images: {built_tags:?}"); - "".to_string() -} - -fn challenge_image_tags(profile_name: &str, chal: &ChallengeConfig) -> Vec { - let config = get_config().unwrap(); - - chal.pods - .iter() - .map(|image| { - format!( - "{registry}/{challenge}-{container}:{profile}", - registry = config.registry.domain, - challenge = chal.name, - container = image.name, - profile = profile_name - ) - }) - .collect() + return Ok(built_tags); } diff --git a/src/commands/build.rs b/src/commands/build.rs index 8bc60c8..3c86093 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -7,5 +7,12 @@ use crate::configparser::{get_config, get_profile_config}; pub fn run(profile_name: &str, push: &bool) { info!("building images..."); - build_challenges(profile_name); + let tags = match build_challenges(profile_name) { + Ok(tags) => tags, + Err(e) => { + error!("{e:?}"); + exit(1) + } + }; + info!("images built successfully!"); } diff --git a/tests/repo/rcds.yaml b/tests/repo/rcds.yaml index 53f823d..8369ca6 100644 --- a/tests/repo/rcds.yaml +++ b/tests/repo/rcds.yaml @@ -1,7 +1,7 @@ flag_regex: dam{[a-zA-Z...]} registry: - domain: registry.example.com/damctf + domain: registry.localhost:5000/damctf # then environment variables e.g. REG_USER/REG_PASS build: user: admin From d5790ecf418f8b79614b0b59653768f3c2156d02 Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Thu, 17 Oct 2024 23:24:32 -0700 Subject: [PATCH 08/10] Move cli help messages to doc comments instead of explicit help messages Makes more sense for these to be comments. Signed-off-by: Robert Detjens --- src/cli.rs | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 8ecc7c2..c15a37f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -17,14 +17,14 @@ pub enum Commands { /// /// Images are tagged as /-:. Build { - #[arg(short, long, value_name = "PROFILE", help = "deployment profile")] + /// Deployment profile + #[arg(short, long, value_name = "PROFILE")] profile: String, - #[arg( - long, - help = "Whether to push container images to registry (default: true)", - default_value = "true" - )] + /// Whether to push container images to registry (default: true) + #[arg(long, default_value = "true")] + // TODO: no way to actually set False... + // maybe revisit when negation flags are implemented: https://github.com/clap-rs/clap/issues/815 push: bool, }, @@ -32,13 +32,16 @@ pub enum Commands { /// /// Also builds and pushes images to registry, unless --no-build is specified. Deploy { - #[arg(short, long, value_name = "PROFILE", help = "deployment profile")] + /// Deployment profile + #[arg(short, long, value_name = "PROFILE")] profile: String, - #[arg(long, help = "Whether to not build/deploy challenge images")] + /// Whether to not build/deploy challenge images + #[arg(long)] no_build: bool, - #[arg(short = 'n', long, help = "Test changes without actually applying")] + /// Test changes without actually applying + #[arg(short = 'n', long)] dry_run: bool, }, @@ -47,22 +50,20 @@ pub enum Commands { /// Checks access to various frontend/backend components. CheckAccess { - #[arg( - short, - long, - value_name = "PROFILE", - help = "deployment profile to check", - default_value = "all" - )] + /// Deployment profile to check + #[arg(short, long, value_name = "PROFILE", default_value = "all")] profile: String, - #[arg(short, long, help = "Check Kubernetes cluster access")] + /// Check Kubernetes cluster access + #[arg(short, long)] kubernetes: bool, - #[arg(short, long, help = "Check frontend (rCTF) access")] + /// Check frontend (rCTF) access + #[arg(short, long)] frontend: bool, - #[arg(short, long, help = "Check container registry access and permissions")] + /// Check container registry access and permissions + #[arg(short, long)] registry: bool, }, } From 5ef9a9d4fd518c4442ed43145a84aa0ef1ef6c8f Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Thu, 17 Oct 2024 23:41:47 -0700 Subject: [PATCH 09/10] implement image pushing and --push/--no-push for build command Signed-off-by: Robert Detjens --- src/builder/docker.rs | 43 ++++++++++++++++++++++++++++++++++++++++++- src/builder/mod.rs | 20 +++++++++++++++----- src/cli.rs | 11 +++++++---- src/commands/build.rs | 14 +++++++++++++- src/main.rs | 9 +++++++-- tests/repo/rcds.yaml | 2 +- 6 files changed, 85 insertions(+), 14 deletions(-) diff --git a/src/builder/docker.rs b/src/builder/docker.rs index 4c43227..0539760 100644 --- a/src/builder/docker.rs +++ b/src/builder/docker.rs @@ -71,7 +71,48 @@ pub async fn build_image(context: &Path, options: &BuildObject, tag: &str) -> Re } } - Ok("".to_string()) + Ok(tag.to_string()) +} + +#[tokio::main(flavor = "current_thread")] // make this a sync function +pub async fn push_image(image_tag: &str, creds: &UserPass) -> Result { + info!("pushing image {image_tag:?} to registry"); + let client = client() + .await + // truncate error chain with new error (returned error is way too verbose) + .map_err(|_| anyhow!("could not talk to Docker daemon (is DOCKER_HOST correct?)"))?; + + let (image, tag) = image_tag + .rsplit_once(":") + .context("failed to get tag from full image string")?; + + let opts = PushImageOptions { tag }; + let creds = DockerCredentials { + username: Some(creds.user.clone()), + password: Some(creds.pass.clone()), + ..Default::default() + }; + + let mut push_stream = client.push_image(image, Some(opts), Some(creds)); + + // stream output to stdout + while let Some(item) = push_stream.next().await { + match item { + // error from stream? + Err(DockerError::DockerResponseServerError { + status_code, + message, + }) => bail!("error from daemon: {message}"), + Err(e) => bail!("{e:?}"), + Ok(msg) => { + debug!("{msg:?}"); + if let Some(progress) = msg.progress_detail { + info!("progress: {:?}/{:?}", progress.current, progress.total); + } + } + } + } + Ok(tag.to_string()) } // diff --git a/src/builder/mod.rs b/src/builder/mod.rs index 6a6c82b..c47701a 100644 --- a/src/builder/mod.rs +++ b/src/builder/mod.rs @@ -48,8 +48,7 @@ fn build_challenge_images(profile_name: &str, chal: &ChallengeConfig) -> Result< debug!("building images for chal {:?}", chal.directory); let config = get_config()?; - let built_tags = chal - .pods + chal.pods .iter() .filter_map(|p| match &p.image_source { Image(_) => None, @@ -72,9 +71,20 @@ fn build_challenge_images(profile_name: &str, chal: &ChallengeConfig) -> Result< ) } }) - .collect::>()?; + .collect::>() +} + +/// Push passed tags to registry +pub fn push_tags(tags: Vec) -> Result> { + let config = get_config()?; - trace!("built these images: {built_tags:?}"); + let built_tags = tags + .iter() + .map(|tag| { + push_image(tag, &config.registry.build) + .with_context(|| format!("error pushing image {tag}")) + }) + .collect::>()?; - return Ok(built_tags); + Ok(built_tags) } diff --git a/src/cli.rs b/src/cli.rs index c15a37f..a232e13 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -20,12 +20,15 @@ pub enum Commands { /// Deployment profile #[arg(short, long, value_name = "PROFILE")] profile: String, - - /// Whether to push container images to registry (default: true) + /// Push container images to registry (default: true) #[arg(long, default_value = "true")] - // TODO: no way to actually set False... - // maybe revisit when negation flags are implemented: https://github.com/clap-rs/clap/issues/815 push: bool, + + /// Don't push container images to registry + #[arg(long, default_value = "false")] + no_push: bool, + // TODO: this is hacky. revisit when automatic negation flags are implemented: + // https://github.com/clap-rs/clap/issues/815 }, /// Deploy enabled challenges to cluster, updating any backing resources as necessary. diff --git a/src/commands/build.rs b/src/commands/build.rs index 3c86093..e4c1852 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -1,7 +1,7 @@ use simplelog::*; use std::process::exit; -use crate::builder::build_challenges; +use crate::builder::{build_challenges, push_tags}; use crate::configparser::{get_config, get_profile_config}; pub fn run(profile_name: &str, push: &bool) { @@ -15,4 +15,16 @@ pub fn run(profile_name: &str, push: &bool) { } }; info!("images built successfully!"); + + if *push { + info!("pushing images..."); + + match push_tags(tags) { + Ok(_) => info!("images pushed successfully!"), + Err(e) => { + error!("{e:?}"); + exit(1) + } + } + }; } diff --git a/src/main.rs b/src/main.rs index 7bf7383..ab1910f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,9 +35,14 @@ fn main() { commands::check_access::run(profile, kubernetes, frontend, registry) } - cli::Commands::Build { profile, push } => { + #[allow(unused_variables)] + cli::Commands::Build { + profile, + push, + no_push, + } => { commands::validate::run(); - commands::build::run(profile, push) + commands::build::run(profile, &!no_push) } cli::Commands::Deploy { diff --git a/tests/repo/rcds.yaml b/tests/repo/rcds.yaml index 8369ca6..ff035c9 100644 --- a/tests/repo/rcds.yaml +++ b/tests/repo/rcds.yaml @@ -1,7 +1,7 @@ flag_regex: dam{[a-zA-Z...]} registry: - domain: registry.localhost:5000/damctf + domain: localhost:5000/damctf # then environment variables e.g. REG_USER/REG_PASS build: user: admin From 147a138d2882aca64445fda65546ea9e8dfbb919 Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sun, 27 Oct 2024 14:22:35 -0700 Subject: [PATCH 10/10] clippy fixes Signed-off-by: Robert Detjens --- src/builder/docker.rs | 7 +++---- src/commands/validate.rs | 15 ++++++--------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/builder/docker.rs b/src/builder/docker.rs index 0539760..b8a9015 100644 --- a/src/builder/docker.rs +++ b/src/builder/docker.rs @@ -58,14 +58,13 @@ pub async fn build_image(context: &Path, options: &BuildObject, tag: &str) -> Re ) } - match msg.stream { - Some(log) => info!( + if let Some(log) = msg.stream { + info!( "building {}: {}", context.to_string_lossy(), // tag, log.trim() - ), - None => (), + ) } } } diff --git a/src/commands/validate.rs b/src/commands/validate.rs index 07167e0..103d773 100644 --- a/src/commands/validate.rs +++ b/src/commands/validate.rs @@ -30,7 +30,7 @@ pub fn run() { // check global deploy settings for invalid challenges info!("validating deploy config..."); for (profile_name, _pconfig) in config.profiles.iter() { - // get em + // fetch from config let deploy_challenges = match get_profile_deploy(profile_name) { Ok(d) => &d.challenges, Err(err) => { @@ -39,18 +39,15 @@ pub fn run() { } }; - // check em + // check for missing let missing: Vec<_> = deploy_challenges .keys() - .filter_map( - // invert match to filter for challenges that *dont* match - |path| match chals.iter().find(|c| c.directory == Path::new(path)) { - Some(_) => None, - None => Some(path), - }, + .filter( + // try to find any challenge paths in deploy config that do not exist + |path| !chals.iter().any(|c| c.directory == Path::new(path)), ) .collect(); - if missing.len() > 0 { + if !missing.is_empty() { error!( "Deploy settings for profile '{profile_name}' has challenges that do not exist:" );