diff --git a/.github/workflows/build_and_upload.yml b/.github/workflows/build_and_upload.yml index 951808a3..7f726941 100644 --- a/.github/workflows/build_and_upload.yml +++ b/.github/workflows/build_and_upload.yml @@ -56,6 +56,7 @@ jobs: # in the future we'll want to release against different architectures matrix: include: + # Linux x86, statically compiled with musl - runs-on: ubuntu-latest # todo: deprecate plaform key when we move to RFC 20 style abq uploads deprecated-platform: linux_x86-64 @@ -63,6 +64,17 @@ jobs: architecture: x86_64 cross-target: 'x86_64-unknown-linux-musl' install-musl-tools: true + # Linux x86, dynamically compiled against glibc. Manylinux 2014 + # provides a sufficiently-old glibc suitable for later linking in + # either EC2 or GHA. + - runs-on: ubuntu-latest + # todo: deprecate plaform key when we move to RFC 20 style abq uploads + deprecated-platform: linux_x86-64_glibc + os: linux + architecture: x86_64_glibc + cross-target: '' + install-musl-tools: false + container: quay.io/pypa/manylinux2014_x86_64 - runs-on: ubuntu-latest deprecated-platform: linux_aarch64 os: linux diff --git a/.github/workflows/test_and_package_development.yml b/.github/workflows/test_and_package_development.yml index ed928207..55898644 100644 --- a/.github/workflows/test_and_package_development.yml +++ b/.github/workflows/test_and_package_development.yml @@ -22,14 +22,14 @@ on: jobs: release: name: Build, Upload, Release ABQ - # We only need x86-64 statically-linked binaries for development testing. + # We only need x86-64 many-glibc binaries for development testing. runs-on: ubuntu-latest + container: quay.io/pypa/manylinux2014_x86_64 outputs: abq_version: ${{ steps.abq_version.outputs.abq_version }} env: AWS_DEFAULT_REGION: us-east-2 AWS_DEFAULT_OUTPUT: json - RUST_TARGET: x86_64-unknown-linux-musl ABQ_DEVELOPMENT_BUILD: 1 ABQ_NO_REBUILD: true steps: @@ -50,13 +50,13 @@ jobs: - name: re-fetch overwritten tag run: git fetch --tags --force + - name: Install build + deploy deps + run: | + yum install wget sudo jq -y + - name: check for typos uses: crate-ci/typos@master - - name: Install musl libc tools - run: | - sudo apt-get install -y musl-tools - - name: Restore cargo cache uses: actions/cache@v3.3.1 env: @@ -68,14 +68,13 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: cargo-release-unstable-1-linux_x86-64-musl-${{ hashFiles('rust-toolchain.toml')}}-${{ hashFiles('Cargo.lock') }} + key: cargo-release-unstable-1-linux_x86-64-manylinux-${{ hashFiles('rust-toolchain.toml')}}-${{ hashFiles('Cargo.lock') }} restore-keys: | - cargo-release-unstable-1-linux_x86-64-musl-${{ hashFiles('rust-toolchain.toml')}}- + cargo-release-unstable-1-linux_x86-64-manylinux-${{ hashFiles('rust-toolchain.toml')}}- - name: Install Rust toolchain uses: rwx-research/rust-toolchain@abq with: toolchain: 1.65.0 - target: ${{ env.RUST_TARGET }} components: clippy, rustfmt - name: Check formatting @@ -88,7 +87,7 @@ jobs: uses: actions-rs/cargo@v1.0.3 with: command: clippy - args: --workspace --tests --profile=release-unstable --all-features --target=${{ env.RUST_TARGET }} -- --deny warnings + args: --workspace --tests --profile=release-unstable --all-features -- --deny warnings - name: Lint github workflows run: | @@ -112,13 +111,13 @@ jobs: - name: Build release with tests run: | - cargo build --tests --bins --profile=release-unstable --all-features --target="$RUST_TARGET" + cargo build --tests --bins --profile=release-unstable --all-features - name: Run tests with cargo timeout-minutes: 4 if: github.run_attempt == 1 run: | - cargo test --profile=release-unstable --all-features --target="$RUST_TARGET" + cargo test --profile=release-unstable --all-features env: CI: false @@ -127,10 +126,16 @@ jobs: if: github.run_attempt > 1 run: | export ABQ_DEBUG_CLI_TESTS_FOR_CI=1 - cargo test --profile=release-unstable --all-features --target="$RUST_TARGET" -- --nocapture + cargo test --profile=release-unstable --all-features -- --nocapture env: CI: false + - name: Install aws CLI + run: | + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + - name: Prepare release environment run: | ABQ_VERSION="$(cat build_artifact/abq_version.txt)" @@ -154,8 +159,8 @@ jobs: id: prepare_release run: | mkdir "${RELEASE_DIR}" - cp "target/$RUST_TARGET/release-unstable/abq" "${RELEASE_DIR}" - cp "target/$RUST_TARGET/release-unstable/abq_tester_harness" "${RELEASE_DIR}" + cp "target/release-unstable/abq" "${RELEASE_DIR}" + cp "target/release-unstable/abq_tester_harness" "${RELEASE_DIR}" tar -czvf "${RELEASE_ARCHIVE}" "${RELEASE_DIR}" - name: Push release to staging S3 diff --git a/Cargo.lock b/Cargo.lock index 3b2b8f32..3b93a039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,7 +30,6 @@ dependencies = [ "indicatif", "indoc", "insta", - "mimalloc", "nix", "ntest", "num_cpus", @@ -1575,16 +1574,6 @@ version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" -[[package]] -name = "libmimalloc-sys" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8c7cbf8b89019683667e347572e6d55a7df7ea36b0c4ce69961b0cde67b174" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1673,15 +1662,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mimalloc" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcb174b18635f7561a0c6c9fc2ce57218ac7523cf72c50af80e2d79ab8f3ba1" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "mime" version = "0.3.16" diff --git a/crates/abq_cli/Cargo.toml b/crates/abq_cli/Cargo.toml index c94ec622..a747bcf0 100644 --- a/crates/abq_cli/Cargo.toml +++ b/crates/abq_cli/Cargo.toml @@ -43,9 +43,6 @@ tracing-appender.workspace = true indoc.workspace = true -[target.x86_64-unknown-linux-musl.dependencies] -mimalloc = "0.1.34" - [dev-dependencies] abq_utils = { path = "../abq_utils", features = ["expose-native-protocols"] } abq_test_utils = { path = "../abq_test_support/abq_test_utils" } diff --git a/crates/abq_cli/src/main.rs b/crates/abq_cli/src/main.rs index d1b56318..922f7e41 100644 --- a/crates/abq_cli/src/main.rs +++ b/crates/abq_cli/src/main.rs @@ -39,10 +39,6 @@ use crate::{ reporting::StdoutPreferences, }; -#[cfg(all(target_arch = "x86_64", target_env = "musl"))] -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - fn main() -> anyhow::Result<()> { let exit_code = abq_main()?; std::process::exit(exit_code.get()); diff --git a/crates/abq_cli/tests/cli.rs b/crates/abq_cli/tests/cli.rs index 9972d187..df82c269 100644 --- a/crates/abq_cli/tests/cli.rs +++ b/crates/abq_cli/tests/cli.rs @@ -384,11 +384,11 @@ macro_rules! setup_queue { test_all_network_config_options! { #[cfg(feature = "test-abq-jest")] - yarn_jest_auto_workers_without_failure |name, conf: CSConfigOptions| { - // abq test --reporter dot (--token ...)? -- yarn jest + npm_jest_auto_workers_without_failure |name, conf: CSConfigOptions| { + // abq test --reporter dot (--token ...)? -- npm test let args = &["test", "--reporter", "dot", "-n", "cpu-cores", "--color=never"]; let mut args = conf.extend_args_for_in_band_client(args); - args.extend(["--", "yarn", "jest"]); + args.extend(["--", "npm", "test"]); let CmdOutput { stdout, stderr: _, @@ -405,11 +405,11 @@ test_all_network_config_options! { test_all_network_config_options! { #[cfg(feature = "test-abq-jest")] - yarn_jest_auto_workers_without_failure_worker_0 |name, conf: CSConfigOptions| { - // abq test --worker 0 --reporter dot (--token ...)? -- yarn jest + npm_jest_auto_workers_without_failure_worker_0 |name, conf: CSConfigOptions| { + // abq test --worker 0 --reporter dot (--token ...)? -- npm test let args = &["test", "--worker", "0", "--reporter", "dot", "-n", "cpu-cores", "--color=never"]; let mut args = conf.extend_args_for_in_band_client(args); - args.extend(["--", "yarn", "jest"]); + args.extend(["--", "npm", "test"]); let CmdOutput { stdout, stderr: _, @@ -506,7 +506,7 @@ fn assert_sum_of_run_test_retries<'a>(outputs: impl IntoIterator test_all_network_config_options! { #[cfg(feature = "test-abq-jest")] - yarn_jest_separate_queue_numbered_workers_test_without_failure |name, conf: CSConfigOptions| { + npm_jest_separate_queue_numbered_workers_test_without_failure |name, conf: CSConfigOptions| { let (queue_proc, queue_addr) = setup_queue!(name, conf); let run_id = RunId::unique().to_string(); @@ -515,7 +515,7 @@ test_all_network_config_options! { let working_dir = npm_jest_project_path.display(); - // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest + // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm test let test_args = |worker: usize| { let args = vec![ format!("test"), @@ -528,7 +528,7 @@ test_all_network_config_options! { format!("--color=never"), ]; let mut args = conf.extend_args_for_client(args); - args.extend([s!("--"), s!("yarn"), s!("jest")]); + args.extend([s!("--"), s!("npm"), s!("test")]); args }; @@ -579,7 +579,7 @@ test_all_network_config_options! { test_all_network_config_options! { #[cfg(feature = "test-abq-jest")] - yarn_jest_timeout_run_workers |name, conf: CSConfigOptions| { + npm_jest_timeout_run_workers |name, conf: CSConfigOptions| { let (queue_proc, queue_addr) = setup_queue!(name, conf); let run_id = RunId::unique().to_string(); @@ -588,7 +588,7 @@ test_all_network_config_options! { let working_dir = npm_jest_project_path.display(); - // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest + // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm test let test_args = |worker: usize| { let args = vec![ format!("test"), @@ -601,7 +601,7 @@ test_all_network_config_options! { format!("--inactivity-timeout-seconds=0"), ]; let mut args = conf.extend_args_for_client(args); - args.extend([s!("--"), s!("yarn"), s!("jest")]); + args.extend([s!("--"), s!("npm"), s!("test")]); args }; @@ -619,8 +619,8 @@ test_all_network_config_options! { test_all_network_config_options! { #[cfg(feature = "test-abq-jest")] - yarn_jest_auto_workers_with_failing_tests |name, conf: CSConfigOptions| { - // abq test --reporter dot (--token ...)? -- yarn jest + npm_jest_auto_workers_with_failing_tests |name, conf: CSConfigOptions| { + // abq test --reporter dot (--token ...)? -- npm test let test_args = &[ "test", "--reporter", @@ -631,7 +631,7 @@ test_all_network_config_options! { "--worker=0", ]; let mut test_args = conf.extend_args_for_in_band_client(test_args); - test_args.extend(["--", "yarn", "jest"]); + test_args.extend(["--", "npm", "test"]); let CmdOutput { stdout, stderr: _, @@ -652,7 +652,7 @@ test_all_network_config_options! { test_all_network_config_options! { #[cfg(feature = "test-abq-jest")] - yarn_jest_separate_queue_workers_with_failing_tests |name, conf: CSConfigOptions| { + npm_jest_separate_queue_workers_with_failing_tests |name, conf: CSConfigOptions| { let (queue_proc, queue_addr) = setup_queue!(name, conf); let run_id = RunId::unique().to_string(); @@ -660,7 +660,7 @@ test_all_network_config_options! { let npm_jest_project_path = testdata_project("jest/npm-jest-project-with-failures"); let working_dir = npm_jest_project_path.display(); - // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest + // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm test let test_args = |worker: usize| { let args = vec![ format!("test"), @@ -673,7 +673,7 @@ test_all_network_config_options! { format!("--color=never"), ]; let mut args = conf.extend_args_for_client(args); - args.extend([s!("--"), s!("yarn"), s!("jest")]); + args.extend([s!("--"), s!("npm"), s!("test")]); args }; let test1_proc = Abq::new(name.to_string() + "_test1").args(test_args(1)).spawn(); @@ -2245,7 +2245,7 @@ test_all_network_config_options! { let simulation = [pack_msgs_to_disk(simulation(true)), pack_msgs_to_disk(simulation(false))]; - // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest + // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm jest let test_args = |worker: usize| { let simulator = native_runner_simulation_bin(); let simulation = &simulation[worker]; diff --git a/crates/abq_test_support/abq_test_utils/src/lib.rs b/crates/abq_test_support/abq_test_utils/src/lib.rs index fbd3e30a..f87ce4ae 100644 --- a/crates/abq_test_support/abq_test_utils/src/lib.rs +++ b/crates/abq_test_support/abq_test_utils/src/lib.rs @@ -27,12 +27,13 @@ pub mod color_writer; pub const WORKSPACE: &str = env!("ABQ_WORKSPACE_DIR"); pub fn artifacts_dir() -> PathBuf { - let path = if cfg!(all(target_arch = "x86_64", target_env = "musl")) { - // GHA is using a musl target + let path = if cfg!(all(target_arch = "x86_64")) { + // GHA x86 linux. if cfg!(debug_assertions) { - "target/x86_64-unknown-linux-musl/release-unstable" + // GHA x86 with debug assertions targets the release-unstable profile. + "target/release-unstable" } else { - "target/x86_64-unknown-linux-musl/release" + "target/release" } } else if cfg!(debug_assertions) { "target/debug"