From e7b0dd5d108cc4da44fa8439293cc16b6bdce43a Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:05:44 -0700 Subject: [PATCH 1/3] Remove `repro` test from integration tests (#3560) * Release 8.7.1 (hotfix) (#3459) * Remove the retention policy setting (#3452) --------- Co-authored-by: Cheick Keita * Revert "Release 8.7.1 (hotfix) (#3459)" (#3468) This reverts commit c69deed50e81cc1805f6f82ebb10513a211cbbe2. * Redo 8.7.1 (#3469) * Redo-8.7.1-hotfix --------- Co-authored-by: Cheick Keita * Support custom ado fields that mark work items as duplicate (#3467) * Add field to ado config for checking duplicate work items * Make duplicate fields nullable and add it to python models * Update broken tests * Update docs to include new ado_duplicate_fields property * Update readme with archive message (#3408) Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/proxy-manager (#3425) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/agent (#3424) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Remove unnecessary method argument (#3473) * Bump elsa from 1.8.1 to 1.9.0 in /src/agent (#3411) Bumps [elsa](https://github.com/manishearth/elsa) from 1.8.1 to 1.9.0. - [Commits](https://github.com/manishearth/elsa/compare/v1.8.1...v1.9.0) --- updated-dependencies: - dependency-name: elsa dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/agent (#3437) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/proxy-manager (#3436) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Updating requirements.txt to accept >= onefuzztypes. (#3477) * Updating requirements.txt to accept >= onefuzztypes. * Trying to loosen restriction. * Bump notify from 6.0.1 to 6.1.1 in /src/agent (#3435) Bumps [notify](https://github.com/notify-rs/notify) from 6.0.1 to 6.1.1. - [Release notes](https://github.com/notify-rs/notify/releases) - [Changelog](https://github.com/notify-rs/notify/blob/main/CHANGELOG.md) - [Commits](https://github.com/notify-rs/notify/compare/notify-6.0.1...notify-6.1.1) --- updated-dependencies: - dependency-name: notify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure_* crates (#3478) * Release 8.8.0 (#3466) * Release 8.8.0 * Bump clap from 4.3.21 to 4.4.2 in /src/agent (#3484) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump gimli from 0.27.3 to 0.28.0 in /src/agent (#3414) Bumps [gimli](https://github.com/gimli-rs/gimli) from 0.27.3 to 0.28.0. - [Changelog](https://github.com/gimli-rs/gimli/blob/master/CHANGELOG.md) - [Commits](https://github.com/gimli-rs/gimli/compare/0.27.3...0.28.0) --- updated-dependencies: - dependency-name: gimli dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump clap from 4.3.21 to 4.4.2 in /src/proxy-manager (#3474) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump winreg from 0.50.0 to 0.51.0 in /src/agent (#3434) Bumps [winreg](https://github.com/gentoo90/winreg-rs) from 0.50.0 to 0.51.0. - [Release notes](https://github.com/gentoo90/winreg-rs/releases) - [Changelog](https://github.com/gentoo90/winreg-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/gentoo90/winreg-rs/compare/v0.50.0...v0.51.0) --- updated-dependencies: - dependency-name: winreg dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Starting integration tests (#3438) * Starting integration tests * Ready to test the test * Parametrize test * checkpoint * Test works * Run integration tests in pipeline * fmt * . * -p * Install clang * quotes not required in yaml? * Hopefully fixed windows? * Try without killondrop * lint * small test * another test * Reuse core name * Wrong step * bump tokio? * Try with rust * make build happy * Bump pete and small clean up * Clean up and make the test pass regularly * fix broken ci * Lower the poll timeout * Set the timeout in a nicer way * fix windows * fmt * Include and copy pdbs * Ignore if pdb is missing on linux * It takes too long for coverage to be generated * lint * Only warn on missing coverage since it's flaky * Fix windows build * Small clean up * Try lowering the poll delay * fix coverage * PR comments * . * Apparently make is missing? * Remove aggressive step skipping in CI * Fix sed checks for CLI versioning (#3486) * Fix sed checks for CLI versioning * Fix. * Fix. * Changing build_cli * Trying greater than * Tring once more. * Trying major minor * trying to replace major minor * Using major minor * Bump bytes from 1.4.0 to 1.5.0 in /src/agent (#3488) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Improve area/iteration path validation (#3489) * Add more comprehensive checks and better error messages to area/iteration path validation * Join invalid chars with space instead of comma * Make tree path validation more testable * Add error code for invalid ADO project in config * Write unit tests for tree path validation * Format tree path unit tests * Merge escape character and control character checks and clarify error message * Improve handling of unexpected breakpoints (#3493) * Improve handling of unexpected breakpoints * fmt * Update azure_* crates (#3503) * Fuzz coverage recording (#3322) * Fuzz coverage recording * Update cargo.toml * Update src/agent/coverage/fuzz/fuzz_targets/fuzz_target_record_coverage.rs Co-authored-by: George Pollard * Fix fuzz --------- Co-authored-by: George Pollard * Reporting coverage on task start up (#3502) * Reporting coverage on task start up * Moving metric up. * Remove feature flag from heartbeat metrics. (#3505) * Update archive notice. (#3507) * Add onefuzz service version to job created events (#3504) * Tevoinea/add version checking in local tasks (#3517) * Compare task version to service version * Swallow output when looking for appropriate name * Create directories if they don't exist in the template (#3522) * Create directories if they don't exist in the template * fmt * Support for retention policies on containers (#3501) - [x] ability to specify a retention period on a container, which applies to newly-created blobs - [x] specify default retention periods in templates from CLI side There's a small breaking change to the Python JobHelper class. * Bump rayon from 1.7.0 to 1.8.0 in /src/agent (#3520) Bumps [rayon](https://github.com/rayon-rs/rayon) from 1.7.0 to 1.8.0. - [Changelog](https://github.com/rayon-rs/rayon/blob/master/RELEASES.md) - [Commits](https://github.com/rayon-rs/rayon/compare/rayon-core-v1.7.0...rayon-core-v1.8.0) --- updated-dependencies: - dependency-name: rayon dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump insta from 1.31.0 to 1.32.0 in /src/agent (#3521) Bumps [insta](https://github.com/mitsuhiko/insta) from 1.31.0 to 1.32.0. - [Changelog](https://github.com/mitsuhiko/insta/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/insta/compare/1.31.0...1.32.0) --- updated-dependencies: - dependency-name: insta dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Disable `repro` and `debug` VM CLI commands. (#3494) * Disable and VM CLI commands. * Formatting. * More formatting. * More formatting. * Removing Repro check. * Make modules case insenstive on windows (#3527) * Make modules and coverage allowlist case insensitive on Windows * Tests and fmt * PR comments * fmt * Debugging missing file coverage * fmt * Broken linux test * Add a case insensitive transformer for better perf * cargo fix * Update windows interceptor list (#3528) * Template creation command (#3531) * Tasks are selectable * Almost there * It works * fmt * remove dead code * Remove unnecessary comments * Improve instructions * fix bug * Add some dummy values for paths * Terminate process on timeout in windows for the coverage task (#3529) * Terminate process on timeout in windows for the coverage task * set the timeout before we start the debugger * split the target launch from the debugger initialization wait for the process to finish on a separate thread * fix build * move comments * Ignore regression update when the work item is in some states (#3532) * Ignore regression update when the work item is in some states * format * formatting * don't hide messages in the poison queue * fix typo * update regression logic update test_template to support regression * build fix * mypy fix * build fix * move regression ignore state under ADODuplicateTemplate * replace extend with append * update set_tcp_keepalive * mke mypy happy * copy ADODuplicateTemplate.OnDuplicate.RegressionIgnoreStates * Removing repro test. --------- Signed-off-by: dependabot[bot] Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> Co-authored-by: Cheick Keita Co-authored-by: Kanan B <32438208+kananb@users.noreply.github.com> Co-authored-by: Marc Greisen Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: George Pollard Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com> Co-authored-by: George Pollard --- src/integration-tests/integration-test.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index 15ffcfb9fe..b71b0b7378 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -244,7 +244,7 @@ class Integration(BaseModel): "--test:{extra_setup_dir}", "--write_test_file={extra_output_dir}/test.txt", ], - pool=PoolName("mariner") + pool=PoolName("mariner"), ), "windows-libfuzzer": Integration( template=TemplateType.libfuzzer, @@ -401,10 +401,13 @@ def try_info_get(data: Any) -> None: self.of.pools.create(name, OS.linux) self.logger.info("creating scaleset for pool: %s", name) self.of.scalesets.create( - name, pool_size, region=region, initial_size=pool_size, image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest" + name, + pool_size, + region=region, + initial_size=pool_size, + image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest", ) - class UnmanagedPool: def __init__( self, @@ -644,7 +647,7 @@ def launch( setup = Directory(os.path.join(setup, config.nested_setup_dir)) job: Optional[Job] = None - + job = self.build_job( duration, pool, target, config, setup, target_exe, inputs ) @@ -1277,7 +1280,7 @@ def check_logs_for_errors(self) -> None: if seen_errors: raise Exception("logs included errors") - + def build_pool_name(self, os_type: str) -> PoolName: return PoolName(f"testpool-{os_type}-{self.test_id}") @@ -1592,13 +1595,6 @@ def try_setup(data: Any) -> None: result = tester.check_jobs(poll=True, stop_on_complete_check=True) if not result: raise Exception("jobs failed") - if skip_repro: - self.logger.warning("not testing crash repro") - else: - launch_result, repros = tester.launch_repro() - result = tester.check_repro(repros) - if not (result and launch_result): - raise Exception("repros failed") tester.check_logs_for_errors() From 2f2a64980f8cf71193c16580025c5f6c26dfd6f0 Mon Sep 17 00:00:00 2001 From: Teo Voinea <58236992+tevoinea@users.noreply.github.com> Date: Tue, 10 Oct 2023 13:42:45 -0400 Subject: [PATCH 2/3] Event export serializer (#3543) * Release 8.7.1 (hotfix) (#3459) * Remove the retention policy setting (#3452) --------- Co-authored-by: Cheick Keita * Revert "Release 8.7.1 (hotfix) (#3459)" (#3468) This reverts commit c69deed50e81cc1805f6f82ebb10513a211cbbe2. * Redo 8.7.1 (#3469) * Redo-8.7.1-hotfix --------- Co-authored-by: Cheick Keita * Support custom ado fields that mark work items as duplicate (#3467) * Add field to ado config for checking duplicate work items * Make duplicate fields nullable and add it to python models * Update broken tests * Update docs to include new ado_duplicate_fields property * Update readme with archive message (#3408) Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/proxy-manager (#3425) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/agent (#3424) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Remove unnecessary method argument (#3473) * Bump elsa from 1.8.1 to 1.9.0 in /src/agent (#3411) Bumps [elsa](https://github.com/manishearth/elsa) from 1.8.1 to 1.9.0. - [Commits](https://github.com/manishearth/elsa/compare/v1.8.1...v1.9.0) --- updated-dependencies: - dependency-name: elsa dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/agent (#3437) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/proxy-manager (#3436) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Updating requirements.txt to accept >= onefuzztypes. (#3477) * Updating requirements.txt to accept >= onefuzztypes. * Trying to loosen restriction. * Bump notify from 6.0.1 to 6.1.1 in /src/agent (#3435) Bumps [notify](https://github.com/notify-rs/notify) from 6.0.1 to 6.1.1. - [Release notes](https://github.com/notify-rs/notify/releases) - [Changelog](https://github.com/notify-rs/notify/blob/main/CHANGELOG.md) - [Commits](https://github.com/notify-rs/notify/compare/notify-6.0.1...notify-6.1.1) --- updated-dependencies: - dependency-name: notify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure_* crates (#3478) * Release 8.8.0 (#3466) * Release 8.8.0 * Bump clap from 4.3.21 to 4.4.2 in /src/agent (#3484) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump gimli from 0.27.3 to 0.28.0 in /src/agent (#3414) Bumps [gimli](https://github.com/gimli-rs/gimli) from 0.27.3 to 0.28.0. - [Changelog](https://github.com/gimli-rs/gimli/blob/master/CHANGELOG.md) - [Commits](https://github.com/gimli-rs/gimli/compare/0.27.3...0.28.0) --- updated-dependencies: - dependency-name: gimli dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump clap from 4.3.21 to 4.4.2 in /src/proxy-manager (#3474) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump winreg from 0.50.0 to 0.51.0 in /src/agent (#3434) Bumps [winreg](https://github.com/gentoo90/winreg-rs) from 0.50.0 to 0.51.0. - [Release notes](https://github.com/gentoo90/winreg-rs/releases) - [Changelog](https://github.com/gentoo90/winreg-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/gentoo90/winreg-rs/compare/v0.50.0...v0.51.0) --- updated-dependencies: - dependency-name: winreg dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Starting integration tests (#3438) * Starting integration tests * Ready to test the test * Parametrize test * checkpoint * Test works * Run integration tests in pipeline * fmt * . * -p * Install clang * quotes not required in yaml? * Hopefully fixed windows? * Try without killondrop * lint * small test * another test * Reuse core name * Wrong step * bump tokio? * Try with rust * make build happy * Bump pete and small clean up * Clean up and make the test pass regularly * fix broken ci * Lower the poll timeout * Set the timeout in a nicer way * fix windows * fmt * Include and copy pdbs * Ignore if pdb is missing on linux * It takes too long for coverage to be generated * lint * Only warn on missing coverage since it's flaky * Fix windows build * Small clean up * Try lowering the poll delay * fix coverage * PR comments * . * Apparently make is missing? * Remove aggressive step skipping in CI * Fix sed checks for CLI versioning (#3486) * Fix sed checks for CLI versioning * Fix. * Fix. * Changing build_cli * Trying greater than * Tring once more. * Trying major minor * trying to replace major minor * Using major minor * Bump bytes from 1.4.0 to 1.5.0 in /src/agent (#3488) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Improve area/iteration path validation (#3489) * Add more comprehensive checks and better error messages to area/iteration path validation * Join invalid chars with space instead of comma * Make tree path validation more testable * Add error code for invalid ADO project in config * Write unit tests for tree path validation * Format tree path unit tests * Merge escape character and control character checks and clarify error message * Improve handling of unexpected breakpoints (#3493) * Improve handling of unexpected breakpoints * fmt * Update azure_* crates (#3503) * Fuzz coverage recording (#3322) * Fuzz coverage recording * Update cargo.toml * Update src/agent/coverage/fuzz/fuzz_targets/fuzz_target_record_coverage.rs Co-authored-by: George Pollard * Fix fuzz --------- Co-authored-by: George Pollard * Reporting coverage on task start up (#3502) * Reporting coverage on task start up * Moving metric up. * Remove feature flag from heartbeat metrics. (#3505) * Update archive notice. (#3507) * Add onefuzz service version to job created events (#3504) * Tevoinea/add version checking in local tasks (#3517) * Compare task version to service version * Swallow output when looking for appropriate name * Create directories if they don't exist in the template (#3522) * Create directories if they don't exist in the template * fmt * Support for retention policies on containers (#3501) - [x] ability to specify a retention period on a container, which applies to newly-created blobs - [x] specify default retention periods in templates from CLI side There's a small breaking change to the Python JobHelper class. * Bump rayon from 1.7.0 to 1.8.0 in /src/agent (#3520) Bumps [rayon](https://github.com/rayon-rs/rayon) from 1.7.0 to 1.8.0. - [Changelog](https://github.com/rayon-rs/rayon/blob/master/RELEASES.md) - [Commits](https://github.com/rayon-rs/rayon/compare/rayon-core-v1.7.0...rayon-core-v1.8.0) --- updated-dependencies: - dependency-name: rayon dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump insta from 1.31.0 to 1.32.0 in /src/agent (#3521) Bumps [insta](https://github.com/mitsuhiko/insta) from 1.31.0 to 1.32.0. - [Changelog](https://github.com/mitsuhiko/insta/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/insta/compare/1.31.0...1.32.0) --- updated-dependencies: - dependency-name: insta dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Disable `repro` and `debug` VM CLI commands. (#3494) * Disable and VM CLI commands. * Formatting. * More formatting. * More formatting. * Removing Repro check. * Make modules case insenstive on windows (#3527) * Make modules and coverage allowlist case insensitive on Windows * Tests and fmt * PR comments * fmt * Debugging missing file coverage * fmt * Broken linux test * Add a case insensitive transformer for better perf * cargo fix * Update windows interceptor list (#3528) * Starting to work on custom converter * Add slim serializer for events export * . * Cleanup and better tests * Update feature-flags.bicep Default off --------- Signed-off-by: dependabot[bot] Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> Co-authored-by: Cheick Keita Co-authored-by: Kanan B <32438208+kananb@users.noreply.github.com> Co-authored-by: Marc Greisen Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Co-authored-by: George Pollard Co-authored-by: George Pollard --- src/ApiService/ApiService/FeatureFlags.cs | 1 + .../ApiService/onefuzzlib/Events.cs | 25 +- .../onefuzzlib/WebhookOperations.cs | 13 +- .../onefuzzlib/events/Converters.cs | 76 ++++++ .../Tests/EventExportConverterTests.cs | 219 ++++++++++++++++++ src/ApiService/Tests/OrmModelsTest.cs | 9 +- .../bicep-templates/feature-flags.bicep | 13 ++ 7 files changed, 337 insertions(+), 19 deletions(-) create mode 100644 src/ApiService/ApiService/onefuzzlib/events/Converters.cs create mode 100644 src/ApiService/Tests/EventExportConverterTests.cs diff --git a/src/ApiService/ApiService/FeatureFlags.cs b/src/ApiService/ApiService/FeatureFlags.cs index e74396e882..5a88d7232d 100644 --- a/src/ApiService/ApiService/FeatureFlags.cs +++ b/src/ApiService/ApiService/FeatureFlags.cs @@ -9,4 +9,5 @@ public static class FeatureFlagConstants { public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention"; public const string EnableWorkItemCreation = "EnableWorkItemCreation"; public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies"; + public const string EnableSlimEventSerialization = "EnableSlimEventSerialization"; } diff --git a/src/ApiService/ApiService/onefuzzlib/Events.cs b/src/ApiService/ApiService/onefuzzlib/Events.cs index 0b806c5806..b6343ac95b 100644 --- a/src/ApiService/ApiService/onefuzzlib/Events.cs +++ b/src/ApiService/ApiService/onefuzzlib/Events.cs @@ -35,7 +35,9 @@ public class Events : IEvents { private readonly IContainers _containers; private readonly ICreds _creds; private readonly JsonSerializerOptions _options; + private readonly JsonSerializerOptions _optionsSlim; private readonly JsonSerializerOptions _deserializingFromBlobOptions; + private readonly IOnefuzzContext _context; public Events(ILogger log, IOnefuzzContext context) { _queue = context.Queue; @@ -47,9 +49,12 @@ public class Events : IEvents { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; _options.Converters.Add(new RemoveUserInfo()); + _optionsSlim = new JsonSerializerOptions(_options); + _optionsSlim.Converters.Add(new EventExportConverter()); _deserializingFromBlobOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; + _context = context; } public virtual async Async.Task QueueSignalrEvent(DownloadableEventMessage message) { @@ -58,7 +63,13 @@ public class Events : IEvents { ("event_id", message.EventId.ToString()) }; var ev = new SignalREvent("events", new List() { message }); - var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: _options); + + var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch { + true => _optionsSlim, + false => _options + }; + + var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: opts); if (!queueResult) { _log.AddTags(tags); @@ -155,16 +166,4 @@ public class Events : IEvents { ); } } - - - public class RemoveUserInfo : JsonConverter { - public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - throw new NotSupportedException("reading UserInfo is not supported"); - } - - public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) { - writer.WriteStartObject(); - writer.WriteEndObject(); - } - } } diff --git a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs index cb43a536e5..cc5367b211 100644 --- a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs @@ -20,10 +20,15 @@ public interface IWebhookOperations : IOrm { public class WebhookOperations : Orm, IWebhookOperations { private readonly IHttpClientFactory _httpFactory; + private readonly JsonSerializerOptions _options; + private readonly JsonSerializerOptions _optionsSlim; public WebhookOperations(IHttpClientFactory httpFactory, ILogger log, IOnefuzzContext context) : base(log, context) { _httpFactory = httpFactory; + _options = EntityConverter.GetJsonSerializerOptions(); + _optionsSlim = new JsonSerializerOptions(_options); + _optionsSlim.Converters.Add(new EventExportConverter()); } public async Async.Task SendEvent(DownloadableEventMessage eventMessage) { @@ -139,11 +144,15 @@ public WebhookOperations(IHttpClientFactory httpFactory, ILogger _optionsSlim, + false => _options + }; if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) { var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookMessage, DataVersion: "2.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) }; - data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions()); + data = JsonSerializer.Serialize(eventGridMessage, options: opts); } else { - data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions()); + data = JsonSerializer.Serialize(webhookMessage, options: opts); } string? digest = null; diff --git a/src/ApiService/ApiService/onefuzzlib/events/Converters.cs b/src/ApiService/ApiService/onefuzzlib/events/Converters.cs new file mode 100644 index 0000000000..47050a496d --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/events/Converters.cs @@ -0,0 +1,76 @@ +using System.Collections; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.OneFuzz.Service { + public class RemoveUserInfo : JsonConverter { + public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + throw new NotSupportedException("reading UserInfo is not supported"); + } + + public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) { + writer.WriteStartObject(); + writer.WriteEndObject(); + } + } + + /// + /// THIS IS A WRITE ONLY JSON CONVERTER + ///
+ /// It should only be used when serializing events to be sent outside of the service + /// + public class EventExportConverter : JsonConverter + where T : DownloadableEventMessage { + public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + throw new NotSupportedException("This converter should only be used when serializing event messages to sent outside of the service"); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) { + BoundedSerializer.WriteInternal(writer, value, options); + } + } + + public class BoundedSerializer { + private static HashSet boundedTypes = new HashSet{ + typeof(Guid), + typeof(DateTime), + typeof(int), + typeof(bool), + typeof(float), + typeof(double), + typeof(long), + typeof(char), + typeof(Uri) + }; + + public static void WriteInternal(Utf8JsonWriter writer, object type, JsonSerializerOptions options) { + writer.WriteStartObject(); + var properties = type.GetType().GetProperties(); + foreach (var property in properties) { + if (property.GetValue(type, null) == null + || typeof(IEnumerable).IsAssignableFrom(property.PropertyType) + || type.GetType() == property.PropertyType) { + continue; + } + if (HasBoundedSerialization(property)) { + var serialized = JsonSerializer.Serialize(property.GetValue(type, null), property.PropertyType, options); + if (!string.IsNullOrEmpty(serialized)) { + writer.WritePropertyName(property.Name); + writer.WriteRawValue(serialized); + } + } else if (property.PropertyType.IsClass) { + writer.WritePropertyName(property.Name); + WriteInternal(writer, property.GetValue(type, null)!, options); + } + } + writer.WriteEndObject(); + } + + public static bool HasBoundedSerialization(PropertyInfo propertyInfo) { + return propertyInfo.PropertyType.IsEnum || + boundedTypes.Contains(propertyInfo.PropertyType) || + typeof(IValidatedString).IsAssignableFrom(propertyInfo.PropertyType); + } + } +} diff --git a/src/ApiService/Tests/EventExportConverterTests.cs b/src/ApiService/Tests/EventExportConverterTests.cs new file mode 100644 index 0000000000..4d129c18c0 --- /dev/null +++ b/src/ApiService/Tests/EventExportConverterTests.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using FluentAssertions; +using FsCheck; +using FsCheck.Xunit; +using Microsoft.OneFuzz.Service; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +using Xunit; + + +namespace Tests; + +public class EventExportConverterTests { + enum Color { + Red, + Blue + } + + [Fact] + public void BaseTypesAreBounded() { + var a = new { + guid = Guid.NewGuid(), + date = new DateTime(), + en = Color.Red, + b = 1, + boo = false, + flo = float.Pi, + doub = double.Tau, + lon = long.MinValue, + cha = 'a' + }; + + a.GetType().GetProperties().All(p => BoundedSerializer.HasBoundedSerialization(p)).Should().BeTrue(); + } + + [Fact] + public void StringIsNotBounded() { + var a = new { + bad = "this is not bounded" + }; + + BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("bad")!).Should().BeFalse(); + } + + [Fact] + public void ValidatedStringIsBounded() { + var a = new { + scalesetid = ScalesetId.Parse("abc-123") + }; + + BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("scalesetid")!).Should().BeTrue(); + } + + [Fact] + public void ComplexObjectsAreSerialized() { + var randomGuid = Guid.NewGuid(); + var a = new DownloadableEventMessage( + randomGuid, + EventType.CrashReported, + new EventCrashReported( + new Report( + "https://example.com", + null, + "target.exe", + "crash", + string.Empty, + new List { "this", "is", "a", "stacktrace" }, + string.Empty, + string.Empty, + null, + Guid.NewGuid(), + Guid.NewGuid(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ), + Container.Parse("this-is-a-container"), + "crash-abc123", + null + ), + Guid.NewGuid(), + "onefuzz", + DateTime.Now, + new Uri("https://example.com"), + null + ); + var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + serializerOptions.Converters.Add(new EventExportConverter()); + + var serialized = JsonSerializer.Serialize(a, serializerOptions); + + serialized.Should().NotBeNullOrEmpty(); + serialized.Should().NotContain("stacktrace"); // List is not serialized + serialized.Should().NotContain("crash-abc123"); // string is not serialized + serialized.Should().Contain("this-is-a-container"); // ValidatedString is serialized + serialized.Should().Contain("crash_reported"); // Enum is serialized + serialized.Should().Contain(DateTime.Now.Year.ToString()); // DateTime is serialized + serialized.Should().Contain(randomGuid.ToString()); // Guid id serialized + } + + [Fact] + public void TestWebhookMessage() { + var a = new WebhookMessageEventGrid( + "2.0.0", + "eventsubject", + EventType.JobCreated, + DateTime.Now, + Guid.NewGuid(), + new WebhookMessage( + Guid.NewGuid(), + EventType.JobCreated, + new EventJobCreated( + Guid.NewGuid(), + new JobConfig("some project", "some name", "some build", 1, "some logs"), + null, + "8.0"), + Guid.NewGuid(), + "onefuzz", + Guid.NewGuid(), + DateTime.Now, + new Uri("https://example.com") + ) + ); + + var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + serializerOptions.Converters.Add(new EventExportConverter()); + + var serialized = JsonSerializer.Serialize(a, serializerOptions); + + serialized.Should().Contain("eventsubject"); + serialized.Should().NotContain("some project"); + } + + public class EventExportConverterSerializationTests { + private readonly JsonSerializerOptions _opts = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + public EventExportConverterSerializationTests() { + _ = Arb.Register(); + _opts.Converters.Add(new EventExportConverter()); + } + + void Test(T v) { + // TODO: Try cloning/creating a new serializer options from the existing one? + var serialized = JsonSerializer.Serialize(v, _opts); + var _ = JsonSerializer.Deserialize(serialized); + } + + [Property] + public void EventNodeHeartbeat(EventNodeHeartbeat e) => Test(e); + + + [Property] + public void EventTaskHeartbeat(EventTaskHeartbeat e) => Test(e); + + [Property] + public void EventTaskStopped(EventTaskStopped e) => Test(e); + + [Property] + public void EventInstanceConfigUpdated(EventInstanceConfigUpdated e) => Test(e); + + [Property] + public void EventProxyCreated(EventProxyCreated e) => Test(e); + + [Property] + public void EventProxyDeleted(EventProxyDeleted e) => Test(e); + + [Property] + public void EventProxyFailed(EventProxyFailed e) => Test(e); + + [Property] + public void EventProxyStateUpdated(EventProxyStateUpdated e) => Test(e); + + + [Property] + public void EventCrashReported(EventCrashReported e) => Test(e); + + + [Property] + public void EventRegressionReported(EventRegressionReported e) => Test(e); + + + [Property] + public void EventFileAdded(EventFileAdded e) => Test(e); + + [Property] + public void EventTaskFailed(EventTaskFailed e) => Test(e); + + [Property] + public void EventTaskStateUpdated(EventTaskStateUpdated e) => Test(e); + + [Property] + public void EventScalesetFailed(EventScalesetFailed e) => Test(e); + + [Property] + public void EventScalesetResizeScheduled(EventScalesetResizeScheduled e) => Test(e); + + [Property] + public void EventScalesetStateUpdated(EventScalesetStateUpdated e) => Test(e); + + [Property] + public void EventNodeDeleted(EventNodeDeleted e) => Test(e); + + [Property] + public void EventNodeCreated(EventNodeCreated e) => Test(e); + + [Property] + public void EventMessage(DownloadableEventMessage e) => Test(e); + } +} diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index 956d0c30c5..6828d7f2fd 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -530,12 +530,13 @@ public class OrmJsonSerialization { //Sample function on how repro a failing test run, using Replay //functionality of FsCheck. Feel free to - /* + + [Fact] void Replay() { - var seed = FsCheck.Random.StdGen.NewStdGen(811038773, 297085737); - var p = Prop.ForAll((NotificationTemplate x) => NotificationTemplate(x)); + var seed = FsCheck.Random.StdGen.NewStdGen(1687595065, 297240661); + var p = Prop.ForAll((DownloadableEventMessage x) => EventMessage(x)); p.Check(new Configuration { Replay = seed }); } - */ + } } diff --git a/src/deployment/bicep-templates/feature-flags.bicep b/src/deployment/bicep-templates/feature-flags.bicep index 46fccb0856..cd39edb52f 100644 --- a/src/deployment/bicep-templates/feature-flags.bicep +++ b/src/deployment/bicep-templates/feature-flags.bicep @@ -102,4 +102,17 @@ resource enableContainerRetentionPolicies 'Microsoft.AppConfiguration/configurat } } +resource enableSlimEventSerialization 'Microsoft.AppConfiguration/configurationStores/keyValues@2021-10-01-preview' = { + parent: featureFlags + name: '.appconfig.featureflag~2FEnableSlimEventSerialization' + properties: { + value: string({ + id: 'EnableSlimEventSerialization' + description: 'Enable serializing events as smaller payloads' + enabled: false + }) + contentType: 'application/vnd.microsoft.appconfig.ff+json;charset=utf-8' + } +} + output AppConfigEndpoint string = 'https://${appConfigName}.azconfig.io' From 46bbe5b87fb947596358daf905988f3599c49209 Mon Sep 17 00:00:00 2001 From: George Pollard Date: Wed, 11 Oct 2023 16:43:02 +1300 Subject: [PATCH 3/3] Include all loader errors when failing (#3509) Closes #2736. Include all errors from the loader transcript when looking for missing DLLs. This will help users to analyze the problem, as this information can include missing symbols, etc. --- .../src/bin/dynamic-library.rs | 26 +++++++++++-------- src/agent/dynamic-library/src/linux.rs | 8 +++--- src/agent/dynamic-library/src/windows.rs | 12 +++++++-- src/agent/onefuzz/src/libfuzzer.rs | 16 +++++++----- src/proxy-manager/Cargo.lock | 4 +-- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/agent/dynamic-library/src/bin/dynamic-library.rs b/src/agent/dynamic-library/src/bin/dynamic-library.rs index 97f0b3e7aa..83cbb372c1 100644 --- a/src/agent/dynamic-library/src/bin/dynamic-library.rs +++ b/src/agent/dynamic-library/src/bin/dynamic-library.rs @@ -38,7 +38,7 @@ fn main() -> Result<()> { cmd.env("LD_LIBRARY_PATH", path); } - let missing = find_missing(cmd)?; + let (missing, errors) = find_missing(cmd)?; if missing.is_empty() { println!("no missing libraries"); @@ -46,23 +46,27 @@ fn main() -> Result<()> { for lib in missing { println!("missing library: {lib:x?}"); } + + if !errors.is_empty() { + println!(); + + for err in errors { + println!("error: {err}"); + } + } } Ok(()) } #[cfg(target_os = "linux")] -fn find_missing(cmd: Command) -> Result> { - Ok(dynamic_library::linux::find_missing(cmd)? - .drain() - .map(|m| m.name) - .collect()) +fn find_missing(cmd: Command) -> Result<(Vec, Vec)> { + let (missing, errors) = dynamic_library::linux::find_missing(cmd)?; + Ok((missing.into_iter().map(|m| m.name).collect(), errors)) } #[cfg(target_os = "windows")] -fn find_missing(cmd: Command) -> Result> { - Ok(dynamic_library::windows::find_missing(cmd)? - .into_iter() - .map(|m| m.name) - .collect()) +fn find_missing(cmd: Command) -> Result<(Vec, Vec)> { + let (missing, errors) = dynamic_library::windows::find_missing(cmd)?; + Ok((missing.into_iter().map(|m| m.name).collect(), errors)) } diff --git a/src/agent/dynamic-library/src/linux.rs b/src/agent/dynamic-library/src/linux.rs index f438e3fa58..98742c54a8 100644 --- a/src/agent/dynamic-library/src/linux.rs +++ b/src/agent/dynamic-library/src/linux.rs @@ -13,7 +13,9 @@ use regex::Regex; const LD_LIBRARY_PATH: &str = "LD_LIBRARY_PATH"; -pub fn find_missing(mut cmd: Command) -> Result, io::Error> { +pub fn find_missing( + mut cmd: Command, +) -> Result<(HashSet, Vec), io::Error> { // Check for missing _linked_ dynamic libraries. // // We must do this first to avoid false positives or negatives when parsing `LD_DEBUG` @@ -24,7 +26,7 @@ pub fn find_missing(mut cmd: Command) -> Result, let missing_linked = linked.not_found(); if !missing_linked.is_empty() { - return Ok(missing_linked); + return Ok((missing_linked, Vec::new())); } // Check for missing _loaded_ dynamic libraries. @@ -34,7 +36,7 @@ pub fn find_missing(mut cmd: Command) -> Result, let output = cmd.output()?; let logs = LdDebugLogs::parse(&*output.stderr); - Ok(logs.missing()) + Ok((logs.missing(), Vec::new())) } pub fn get_linked_library_logs(cmd: &Command) -> Result { diff --git a/src/agent/dynamic-library/src/windows.rs b/src/agent/dynamic-library/src/windows.rs index 137d10ea03..be847357b6 100644 --- a/src/agent/dynamic-library/src/windows.rs +++ b/src/agent/dynamic-library/src/windows.rs @@ -26,10 +26,10 @@ pub enum CheckDynamicLibrariesError { pub fn find_missing( cmd: Command, -) -> Result, CheckDynamicLibrariesError> { +) -> Result<(Vec, Vec), CheckDynamicLibrariesError> { let mut handler = LoaderSnapsHandler::default(); setup_debugger(cmd, &mut handler)?; - Ok(handler.missing_libraries()) + Ok((handler.missing_libraries(), handler.errors())) } pub fn get_logs(cmd: Command) -> Result, CheckDynamicLibrariesError> { @@ -273,6 +273,14 @@ impl LoaderSnapsHandler { missing } + + pub fn errors(&self) -> Vec { + self.debug_strings + .iter() + .filter(|text| text.contains("ERROR:")) + .map(|text| text.to_owned()) + .collect() + } } impl DebugEventHandler for LoaderSnapsHandler { diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index 0d36a45286..87428b102f 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -306,17 +306,18 @@ impl LibFuzzer { if !result.status.success() { // To provide user-actionable errors, try to identify any missing shared libraries. match self.find_missing_libraries().await { - Ok(missing) => { + Ok((missing, errors)) => { if missing.is_empty() { - bail!("fuzzer does not respond to '-help=1'. no missing shared libraries detected. output: {:?}", result); + bail!("fuzzer does not respond to '-help=1'. no missing shared libraries detected. output: {result:?}"); } else { let missing = missing.join(", "); + let all_errors = errors.join("\n"); - bail!("fuzzer does not respond to '-help=1'. missing shared libraries: {}. output: {:?}", missing, result); + bail!("fuzzer does not respond to '-help=1'. missing shared libraries: {missing}. output: {result:?}. all errors: {all_errors}"); } } Err(err) => { - bail!("fuzzer does not respond to '-help=1'. additional error while checking for missing shared libraries: {}. output: {:?}", err, result); + bail!("fuzzer does not respond to '-help=1'. additional error while checking for missing shared libraries: {err}. output: {result:?}"); } } } @@ -324,7 +325,7 @@ impl LibFuzzer { Ok(()) } - async fn find_missing_libraries(&self) -> Result> { + async fn find_missing_libraries(&self) -> Result<(Vec, Vec)> { let cmd = self.build_std_command(None, None, None, None, None)?; #[cfg(target_os = "linux")] @@ -333,10 +334,11 @@ impl LibFuzzer { #[cfg(target_os = "windows")] let blocking = move || dynamic_library::windows::find_missing(cmd); - let missing = tokio::task::spawn_blocking(blocking).await??; + let (missing, errors) = tokio::task::spawn_blocking(blocking).await??; + let missing = missing.into_iter().map(|m| m.name).collect(); - Ok(missing) + Ok((missing, errors)) } pub fn fuzz( diff --git a/src/proxy-manager/Cargo.lock b/src/proxy-manager/Cargo.lock index e86da294a7..a9a7b85c6d 100644 --- a/src/proxy-manager/Cargo.lock +++ b/src/proxy-manager/Cargo.lock @@ -199,9 +199,9 @@ checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" dependencies = [ "serde", ]