import trace data from cloud providers

ctrl/controller.rs

@@ -199,7 +199,7 @@ pub(super) async fn setup_simulation(
     let jobs_api = kube::Api::<batchv1::Job>::namespaced(ctx.client.clone(), &sim.spec.driver.namespace);
     if jobs_api.get_opt(&ctx.driver_name).await?.is_none() {
         info!("creating simulation driver {}", ctx.driver_name);
-        let obj = build_driver_job(ctx, sim, &driver_cert_secret_name, ctrl_ns)?;
+        let obj = build_driver_job(ctx, sim, ctx.opts.driver_secrets.as_ref(), &driver_cert_secret_name, ctrl_ns)?;
         jobs_api.create(&Default::default(), &obj).await?;
     }
 

ctrl/main.rs

@@ -1,7 +1,6 @@
 mod cert_manager;
 mod controller;
 mod objects;
-mod trace;
 
 use std::ops::Deref;
 use std::sync::Arc;
@@ -30,6 +29,9 @@ use crate::controller::{
 
 #[derive(Clone, Debug, Parser)]
 struct Options {
+    #[arg(long, value_delimiter = ',')]
+    driver_secrets: Option<Vec<String>>,
+
     // TODO: should support non-cert-manager for configuring certs as well
     #[arg(long)]
     use_cert_manager: bool,

ctrl/objects.rs

@@ -1,5 +1,7 @@
 use std::env;
+use std::path::PathBuf;
 
+use anyhow::anyhow;
 use k8s_openapi::api::admissionregistration::v1 as admissionv1;
 use k8s_openapi::api::batch::v1 as batchv1;
 use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
@@ -21,8 +23,7 @@ use simkube::prelude::*;
 use simkube::sim::*;
 use simkube::store::external_storage::ObjectStoreScheme;
 
-use super::cert_manager::DRIVER_CERT_NAME;
-use super::trace::get_local_trace_volume;
+use crate::cert_manager::DRIVER_CERT_NAME;
 use crate::SimulationContext;
 
 const METRICS_NAME_LABEL: &str = "__name__";
@@ -31,6 +32,11 @@ const PROM_VERSION: &str = "2.44.0";
 const PROM_COMPONENT_LABEL: &str = "prometheus";
 const WEBHOOK_NAME: &str = "mutatepods.simkube.io";
 const DRIVER_CERT_VOLUME: &str = "driver-cert";
+const TRACE_VOLUME_NAME: &str = "trace-data";
+const TRACE_PATH: &str = "/trace-data";
+const SSL_MOUNT_PATH: &str = "/usr/local/etc/ssl";
+
+type VolumeInfo = (corev1::VolumeMount, corev1::Volume, String);
 
 pub(super) fn build_driver_namespace(ctx: &SimulationContext, sim: &Simulation) -> corev1::Namespace {
     let owner = sim;
@@ -177,19 +183,33 @@ pub(super) fn build_driver_service(
 pub(super) fn build_driver_job(
     ctx: &SimulationContext,
     sim: &Simulation,
+    driver_secrets: Option<&Vec<String>>,
     cert_secret_name: &str,
     ctrl_ns: &str,
 ) -> anyhow::Result<batchv1::Job> {
-    let trace_url = Url::parse(&sim.spec.driver.trace_path)?;
-    let (trace_vm, trace_volume, trace_mount_path) = match ObjectStoreScheme::parse(&trace_url)? {
-        (ObjectStoreScheme::AmazonS3, _) => todo!(),
-        (ObjectStoreScheme::Local, _) => get_local_trace_volume(&trace_url)?,
-        _ => unimplemented!(),
-    };
     let (cert_vm, cert_volume, cert_mount_path) = build_certificate_volumes(cert_secret_name);
+    let (mut volume_mounts, mut volumes) = (vec![cert_vm], vec![cert_volume]);
 
+    let trace_path = match build_local_trace_volume(&sim.spec.driver.trace_path)? {
+        Some((trace_vm, trace_volume, trace_mount_path)) => {
+            volume_mounts.push(trace_vm);
+            volumes.push(trace_volume);
+            trace_mount_path
+        },
+        None => sim.spec.driver.trace_path.clone(),
+    };
     let service_account = Some(env::var(POD_SVC_ACCOUNT_ENV_VAR)?);
 
+    let driver_secret_refs = driver_secrets.as_ref().map(|secrets_list| {
+        secrets_list
+            .iter()
+            .map(|s| corev1::EnvFromSource {
+                secret_ref: Some(corev1::SecretEnvSource { name: Some(s.clone()), optional: Some(false) }),
+                ..Default::default()
+            })
+            .collect()
+    });
+
     Ok(batchv1::Job {
         metadata: build_object_meta(&sim.spec.driver.namespace, &ctx.driver_name, &ctx.name, sim),
         spec: Some(batchv1::JobSpec {
@@ -199,8 +219,9 @@ pub(super) fn build_driver_job(
                     containers: vec![corev1::Container {
                         name: "driver".into(),
                         command: Some(vec!["/sk-driver".into()]),
-                        args: Some(build_driver_args(ctx, cert_mount_path, trace_mount_path, ctrl_ns.into())),
+                        args: Some(build_driver_args(ctx, cert_mount_path, trace_path, ctrl_ns.into())),
                         image: Some(sim.spec.driver.image.clone()),
+                        env_from: driver_secret_refs,
                         env: Some(vec![
                             corev1::EnvVar {
                                 name: "RUST_BACKTRACE".into(),
@@ -219,11 +240,11 @@ pub(super) fn build_driver_job(
                                 ..Default::default()
                             },
                         ]),
-                        volume_mounts: Some(vec![trace_vm, cert_vm]),
+                        volume_mounts: Some(volume_mounts),
                         ..Default::default()
                     }],
                     restart_policy: Some("Never".into()),
-                    volumes: Some(vec![trace_volume, cert_volume]),
+                    volumes: Some(volumes),
                     service_account,
                     ..Default::default()
                 }),
@@ -237,39 +258,19 @@ pub(super) fn build_driver_job(
     })
 }
 
-fn build_certificate_volumes(cert_secret_name: &str) -> (corev1::VolumeMount, corev1::Volume, String) {
-    (
-        corev1::VolumeMount {
-            name: DRIVER_CERT_VOLUME.into(),
-            mount_path: "/etc/ssl/".into(),
-            ..Default::default()
-        },
-        corev1::Volume {
-            name: DRIVER_CERT_VOLUME.into(),
-            secret: Some(corev1::SecretVolumeSource {
-                secret_name: Some(cert_secret_name.into()),
-                default_mode: Some(0o600),
-                ..Default::default()
-            }),
-            ..Default::default()
-        },
-        "/etc/ssl/".into(),
-    )
-}
-
 fn build_driver_args(
     ctx: &SimulationContext,
     cert_mount_path: String,
-    trace_mount_path: String,
+    trace_path: String,
     ctrl_ns: String,
 ) -> Vec<String> {
     vec![
         "--cert-path".into(),
         format!("{cert_mount_path}/tls.crt"),
         "--key-path".into(),
         format!("{cert_mount_path}/tls.key"),
-        "--trace-mount-path".into(),
-        trace_mount_path,
+        "--trace-path".into(),
+        trace_path,
         "--virtual-ns-prefix".into(),
         "virtual".into(),
         "--sim-name".into(),
@@ -280,3 +281,61 @@ fn build_driver_args(
         ctrl_ns,
     ]
 }
+
+fn build_certificate_volumes(cert_secret_name: &str) -> VolumeInfo {
+    (
+        corev1::VolumeMount {
+            name: DRIVER_CERT_VOLUME.into(),
+            mount_path: SSL_MOUNT_PATH.into(),
+            ..Default::default()
+        },
+        corev1::Volume {
+            name: DRIVER_CERT_VOLUME.into(),
+            secret: Some(corev1::SecretVolumeSource {
+                secret_name: Some(cert_secret_name.into()),
+                default_mode: Some(0o600),
+                ..Default::default()
+            }),
+            ..Default::default()
+        },
+        SSL_MOUNT_PATH.into(),
+    )
+}
+
+fn build_local_trace_volume(trace_path: &str) -> anyhow::Result<Option<VolumeInfo>> {
+    let url = Url::parse(trace_path)?;
+    let (scheme, trace_path) = ObjectStoreScheme::parse(&url)?;
+    if scheme != ObjectStoreScheme::Local {
+        return Ok(None);
+    }
+
+    let fp = url
+        .to_file_path()
+        .map_err(|_| anyhow!("could not parse trace path: {}", trace_path))?;
+
+    let host_path_str = fp
+        .clone()
+        .into_os_string()
+        .into_string()
+        .map_err(|osstr| anyhow!("could not parse host path: {:?}", osstr))?;
+
+    let mut mount_path = PathBuf::from(TRACE_PATH);
+    mount_path.push(fp);
+    let mount_path_str = mount_path
+        .to_str()
+        .ok_or(anyhow!("could not parse trace mount path: {}", mount_path.display()))?;
+
+    Ok(Some((
+        corev1::VolumeMount {
+            name: TRACE_VOLUME_NAME.into(),
+            mount_path: mount_path_str.into(),
+            ..Default::default()
+        },
+        corev1::Volume {
+            name: TRACE_VOLUME_NAME.into(),
+            host_path: Some(corev1::HostPathVolumeSource { path: host_path_str, type_: Some("File".into()) }),
+            ..Default::default()
+        },
+        mount_path_str.into(),
+    )))
+}

driver/main.rs

@@ -1,26 +1,28 @@
 mod mutation;
 mod runner;
 
+use std::env;
 use std::net::{
     IpAddr,
     Ipv4Addr,
 };
 use std::sync::Arc;
 use std::time::Duration;
-use std::{
-    env,
-    fs,
-};
 
 use anyhow::anyhow;
 use clap::Parser;
+use reqwest::Url;
 use rocket::config::TlsConfig;
 use simkube::k8s::{
     ApiSet,
     OwnersCache,
 };
 use simkube::prelude::*;
 use simkube::sim::hooks;
+use simkube::store::external_storage::{
+    object_store_for_scheme,
+    ObjectStoreScheme,
+};
 use simkube::store::{
     TraceStorable,
     TraceStore,
@@ -55,7 +57,7 @@ struct Options {
     // because the location the trace is mounted in the pod will be different than
     // the location specified in the spec
     #[arg(long)]
-    trace_mount_path: String,
+    trace_path: String,
 
     #[arg(short, long, default_value = "info")]
     verbosity: String,
@@ -82,7 +84,11 @@ async fn run(opts: Options) -> EmptyResult {
 
     let root_name = format!("{name}-root");
 
-    let trace_data = fs::read(opts.trace_mount_path)?;
+    let url = Url::parse(&opts.trace_path)?;
+    let (scheme, path) = ObjectStoreScheme::parse(&url)?;
+    let store = object_store_for_scheme(&scheme, &opts.trace_path)?;
+    let trace_data = store.get(&path).await?.bytes().await?.to_vec();
+
     let store = Arc::new(TraceStore::import(trace_data, &sim.spec.duration)?);
 
     let apiset = ApiSet::new(client.clone());

k8s/kustomize/0002-sk-ctrl.k8s.yaml

@@ -37,6 +37,8 @@ spec:
       containers:
         - args:
             - /sk-ctrl
+            - --driver-secrets
+            - simkube
             - --use-cert-manager
             - --cert-manager-issuer
             - selfsigned