Starting integration tests (microsoft#3438)

* Starting integration tests

* Ready to test the test

* Parametrize test

* checkpoint

* Test works

* Run integration tests in pipeline

* fmt

* .

* -p

* Install clang

* quotes not required in yaml?

* Hopefully fixed windows?

* Try without killondrop

* lint

* small test

* another test

* Reuse core name

* Wrong step

* bump tokio?

* Try with rust

* make build happy

* Bump pete and small clean up

* Clean up and make the test pass regularly

* fix broken ci

* Lower the poll timeout

* Set the timeout in a nicer way

* fix windows

* fmt

* Include and copy pdbs

* Ignore if pdb is missing on linux

* It takes too long for coverage to be generated

* lint

* Only warn on missing coverage since it's flaky

* Fix windows build

* Small clean up

* Try lowering the poll delay

* fix coverage

* PR comments

* .

* Apparently make is missing?

* Remove aggressive step skipping in CI

.github/workflows/ci.yml

@@ -79,16 +79,24 @@ jobs:
           key: ${{env.ACTIONS_CACHE_KEY_DATE}} # additional key for cache-busting
           workspaces: src/agent
       - name: Linux Prereqs
-        if: runner.os == 'Linux' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
+        if: runner.os == 'Linux'
         run: |
           sudo apt-get -y update
-          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config
+          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config clang
+      - name: Clone onefuzz-samples
+        run: git clone https://github.com/microsoft/onefuzz-samples
+      - name: Prepare for agent integration tests
+        shell: bash
+        working-directory: ./onefuzz-samples/examples/simple-libfuzzer
+        run: |
+          make
+          mkdir -p ../../../src/agent/onefuzz-task/tests/targets/simple
+          cp fuzz.exe ../../../src/agent/onefuzz-task/tests/targets/simple/fuzz.exe
+          cp *.pdb ../../../src/agent/onefuzz-task/tests/targets/simple/ 2>/dev/null || :
       - name: Install Rust Prereqs
-        if: steps.rust-build-cache.outputs.cache-hit != 'true' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
         run: src/ci/rust-prereqs.sh
       - run: src/ci/agent.sh
-        if: steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

src/agent/coverage/Cargo.toml

@@ -26,7 +26,7 @@ debugger = { path = "../debugger" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 nix = "0.26"
-pete = "0.10"
+pete = "0.12"
 # For procfs, opt out of the `chrono` freature; it pulls in an old version
 # of `time`. We do not use the methods that the `chrono` feature enables.
 procfs = { version = "0.15.1", default-features = false, features = ["flate2"] }

src/agent/coverage/src/record/linux/debugger.rs

@@ -4,6 +4,7 @@
 use std::collections::BTreeMap;
 use std::io::Read;
 use std::process::{Child, Command};
+use std::time::Duration;
 
 use anyhow::{bail, format_err, Result};
 use debuggable_module::path::FilePath;
@@ -75,7 +76,11 @@ impl<'eh> Debugger<'eh> {
         // These calls should also be unnecessary no-ops, but we really want to avoid any dangling
         // or zombie child processes.
         let _ = child.kill();
-        let _ = child.wait();
+
+        // We don't need to call child.wait() because of the following series of events:
+        // 1. pete, our ptracing library, spawns the child process with ptrace flags
+        // 2. rust stdlib set SIG_IGN as the SIGCHLD handler: https://github.com/rust-lang/rust/issues/110317
+        // 3. linux kernel automatically reaps pids when the above 2 hold: https://github.com/torvalds/linux/blob/44149752e9987a9eac5ad78e6d3a20934b5e018d/kernel/signal.c#L2089-L2110
 
         let output = Output {
             status,
@@ -198,8 +203,8 @@ impl DebuggerContext {
     pub fn new() -> Self {
         let breakpoints = Breakpoints::default();
         let images = None;
-        let tracer = Ptracer::new();
-
+        let mut tracer = Ptracer::new();
+        *tracer.poll_delay_mut() = Duration::from_millis(1);
         Self {
             breakpoints,
             images,

src/agent/onefuzz-task/Cargo.toml

@@ -6,6 +6,14 @@ edition = "2021"
 publish = false
 license = "MIT"
 
+[lib]
+path = "src/lib.rs"
+name = "onefuzz_task_lib"
+
+[[bin]]
+path = "src/main.rs"
+name = "onefuzz-task"
+
 [features]
 integration_test = []
 
@@ -77,3 +85,4 @@ schemars = { version = "0.8.12", features = ["uuid1"] }
 
 [dev-dependencies]
 pretty_assertions = "1.4"
+tempfile = "3.8"

src/agent/onefuzz-task/src/lib.rs

@@ -0,0 +1,9 @@
+#[macro_use]
+extern crate anyhow;
+#[macro_use]
+extern crate clap;
+#[macro_use]
+extern crate onefuzz_telemetry;
+
+pub mod local;
+pub mod tasks;

src/agent/onefuzz-task/src/tasks/coverage/generic.rs

@@ -161,7 +161,7 @@ impl CoverageTask {
         }
 
         if seen_inputs {
-            context.report_coverage_stats().await?;
+            context.report_coverage_stats().await;
             context.save_and_sync_coverage().await?;
         }
 
@@ -454,7 +454,7 @@ impl<'a> TaskContext<'a> {
         Ok(count)
     }
 
-    pub async fn report_coverage_stats(&self) -> Result<()> {
+    pub async fn report_coverage_stats(&self) {
         use EventData::*;
 
         let coverage = RwLock::read(&self.coverage).await;
@@ -471,7 +471,6 @@ impl<'a> TaskContext<'a> {
                 ]),
             )
             .await;
-        Ok(())
     }
 
     pub async fn save_coverage(
@@ -565,7 +564,7 @@ impl<'a> Processor for TaskContext<'a> {
         self.heartbeat.alive();
 
         self.record_input(input).await?;
-        self.report_coverage_stats().await?;
+        self.report_coverage_stats().await;
         self.save_and_sync_coverage().await?;
 
         Ok(())

src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs

@@ -272,7 +272,7 @@ where
         info!("config is: {:?}", self.config);
 
         let fuzzer = L::from_config(&self.config).await?;
-        let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs).await?;
+        let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?;
 
         info!("child is: {:?}", running);
 

src/agent/onefuzz-task/tests/template_integration.rs

@@ -0,0 +1,212 @@
+use std::{
+    collections::HashSet,
+    ffi::OsStr,
+    path::{Path, PathBuf},
+};
+
+use tokio::fs;
+
+use anyhow::Result;
+use log::info;
+use onefuzz_task_lib::local::template;
+use std::time::Duration;
+use tokio::time::timeout;
+
+macro_rules! libfuzzer_tests {
+    ($($name:ident: $value:expr,)*) => {
+        $(
+            #[tokio::test(flavor = "multi_thread")]
+            #[cfg_attr(not(feature = "integration_test"), ignore)]
+            async fn $name() {
+                let _ = env_logger::builder().is_test(true).try_init();
+                let (config, libfuzzer_target) = $value;
+                test_libfuzzer_basic_template(PathBuf::from(config), PathBuf::from(libfuzzer_target)).await;
+            }
+        )*
+    }
+}
+
+// This is the format for adding other templates/targets for this macro
+// $TEST_NAME: ($RELATIVE_PATH_TO_TEMPLATE, $RELATIVE_PATH_TO_TARGET),
+// Make sure that you place the target binary in CI
+libfuzzer_tests! {
+    libfuzzer_basic: ("./tests/templates/libfuzzer_basic.yml", "./tests/targets/simple/fuzz.exe"),
+}
+
+async fn test_libfuzzer_basic_template(config: PathBuf, libfuzzer_target: PathBuf) {
+    assert_exists_and_is_file(&config).await;
+    assert_exists_and_is_file(&libfuzzer_target).await;
+
+    let test_layout = create_test_directory(&config, &libfuzzer_target)
+        .await
+        .expect("Failed to create test directory layout");
+
+    info!("Executed test from: {:?}", &test_layout.root);
+    info!("Running template for 1 minute...");
+    if let Ok(template_result) = timeout(
+        Duration::from_secs(60),
+        template::launch(&test_layout.config, None),
+    )
+    .await
+    {
+        // Something went wrong when running the template so lets print out the template to be helpful
+        info!("Printing config as it was used in the test:");
+        info!("{:?}", fs::read_to_string(&test_layout.config).await);
+        template_result.unwrap();
+    }
+
+    verify_test_layout_structure_did_not_change(&test_layout).await;
+    assert_directory_is_not_empty(&test_layout.inputs).await;
+    assert_directory_is_not_empty(&test_layout.crashes).await;
+    verify_coverage_dir(&test_layout.coverage).await;
+
+    let _ = fs::remove_dir_all(&test_layout.root).await;
+}
+
+async fn verify_test_layout_structure_did_not_change(test_layout: &TestLayout) {
+    assert_exists_and_is_dir(&test_layout.root).await;
+    assert_exists_and_is_file(&test_layout.config).await;
+    assert_exists_and_is_file(&test_layout.target_exe).await;
+    assert_exists_and_is_dir(&test_layout.crashdumps).await;
+    assert_exists_and_is_dir(&test_layout.coverage).await;
+    assert_exists_and_is_dir(&test_layout.crashes).await;
+    assert_exists_and_is_dir(&test_layout.inputs).await;
+    assert_exists_and_is_dir(&test_layout.regression_reports).await;
+}
+
+async fn verify_coverage_dir(coverage: &Path) {
+    warn_if_empty(coverage).await;
+}
+
+async fn assert_exists_and_is_dir(dir: &Path) {
+    assert!(dir.exists(), "Expected directory to exist. dir = {:?}", dir);
+    assert!(
+        dir.is_dir(),
+        "Expected path to be a directory. dir = {:?}",
+        dir
+    );
+}
+
+async fn warn_if_empty(dir: &Path) {
+    if dir_is_empty(dir).await {
+        println!("Expected directory to not be empty: {:?}", dir);
+    }
+}
+
+async fn assert_exists_and_is_file(file: &Path) {
+    assert!(file.exists(), "Expected file to exist. file = {:?}", file);
+    assert!(
+        file.is_file(),
+        "Expected path to be a file. file = {:?}",
+        file
+    );
+}
+
+async fn dir_is_empty(dir: &Path) -> bool {
+    fs::read_dir(dir)
+        .await
+        .unwrap_or_else(|_| panic!("Failed to list files in directory. dir = {:?}", dir))
+        .next_entry()
+        .await
+        .unwrap_or_else(|_| {
+            panic!(
+                "Failed to get next file in directory listing. dir = {:?}",
+                dir
+            )
+        })
+        .is_some()
+}
+
+async fn assert_directory_is_not_empty(dir: &Path) {
+    assert!(
+        dir_is_empty(dir).await,
+        "Expected directory to not be empty. dir = {:?}",
+        dir
+    );
+}
+
+async fn create_test_directory(config: &Path, target_exe: &Path) -> Result<TestLayout> {
+    let mut test_directory = PathBuf::from(".").join(uuid::Uuid::new_v4().to_string());
+    fs::create_dir_all(&test_directory).await?;
+    test_directory = test_directory.canonicalize()?;
+
+    let mut inputs_directory = PathBuf::from(&test_directory).join("inputs");
+    fs::create_dir(&inputs_directory).await?;
+    inputs_directory = inputs_directory.canonicalize()?;
+
+    let mut crashes_directory = PathBuf::from(&test_directory).join("crashes");
+    fs::create_dir(&crashes_directory).await?;
+    crashes_directory = crashes_directory.canonicalize()?;
+
+    let mut crashdumps_directory = PathBuf::from(&test_directory).join("crashdumps");
+    fs::create_dir(&crashdumps_directory).await?;
+    crashdumps_directory = crashdumps_directory.canonicalize()?;
+
+    let mut coverage_directory = PathBuf::from(&test_directory).join("coverage");
+    fs::create_dir(&coverage_directory).await?;
+    coverage_directory = coverage_directory.canonicalize()?;
+
+    let mut regression_reports_directory =
+        PathBuf::from(&test_directory).join("regression_reports");
+    fs::create_dir(&regression_reports_directory).await?;
+    regression_reports_directory = regression_reports_directory.canonicalize()?;
+
+    let mut target_in_test = PathBuf::from(&test_directory).join("fuzz.exe");
+    fs::copy(target_exe, &target_in_test).await?;
+    target_in_test = target_in_test.canonicalize()?;
+
+    let mut interesting_extensions = HashSet::new();
+    interesting_extensions.insert(Some(OsStr::new("so")));
+    interesting_extensions.insert(Some(OsStr::new("pdb")));
+    let mut f = fs::read_dir(target_exe.parent().unwrap()).await?;
+    while let Ok(Some(f)) = f.next_entry().await {
+        if interesting_extensions.contains(&f.path().extension()) {
+            fs::copy(f.path(), PathBuf::from(&test_directory).join(f.file_name())).await?;
+        }
+    }
+
+    let mut config_data = fs::read_to_string(config).await?;
+
+    config_data = config_data
+        .replace("{TARGET_PATH}", target_in_test.to_str().unwrap())
+        .replace("{INPUTS_PATH}", inputs_directory.to_str().unwrap())
+        .replace("{CRASHES_PATH}", crashes_directory.to_str().unwrap())
+        .replace("{CRASHDUMPS_PATH}", crashdumps_directory.to_str().unwrap())
+        .replace("{COVERAGE_PATH}", coverage_directory.to_str().unwrap())
+        .replace(
+            "{REGRESSION_REPORTS_PATH}",
+            regression_reports_directory.to_str().unwrap(),
+        )
+        .replace("{TEST_DIRECTORY}", test_directory.to_str().unwrap());
+
+    let mut config_in_test =
+        PathBuf::from(&test_directory).join(config.file_name().unwrap_or_else(|| {
+            panic!("Failed to get file name for config. config = {:?}", config)
+        }));
+
+    fs::write(&config_in_test, &config_data).await?;
+    config_in_test = config_in_test.canonicalize()?;
+
+    Ok(TestLayout {
+        root: test_directory,
+        config: config_in_test,
+        target_exe: target_in_test,
+        inputs: inputs_directory,
+        crashes: crashes_directory,
+        crashdumps: crashdumps_directory,
+        coverage: coverage_directory,
+        regression_reports: regression_reports_directory,
+    })
+}
+
+#[derive(Debug)]
+struct TestLayout {
+    root: PathBuf,
+    config: PathBuf,
+    target_exe: PathBuf,
+    inputs: PathBuf,
+    crashes: PathBuf,
+    crashdumps: PathBuf,
+    coverage: PathBuf,
+    regression_reports: PathBuf,
+}