Revive manylinux builds, and drop mimalloc

.github/workflows/build_and_upload.yml

@@ -56,13 +56,25 @@ jobs:
       # in the future we'll want to release against different architectures
       matrix:
         include:
+          # Linux x86, statically compiled with musl
           - runs-on: ubuntu-latest
             # todo: deprecate plaform key when we move to RFC 20 style abq uploads
             deprecated-platform: linux_x86-64
             os: linux
             architecture: x86_64
             cross-target: 'x86_64-unknown-linux-musl'
             install-musl-tools: true
+          # Linux x86, dynamically compiled against glibc. Manylinux 2014
+          # provides a sufficiently-old glibc suitable for later linking in
+          # either EC2 or GHA.
+          - runs-on: ubuntu-latest
+            # todo: deprecate plaform key when we move to RFC 20 style abq uploads
+            deprecated-platform: linux_x86-64_glibc
+            os: linux
+            architecture: x86_64_glibc
+            cross-target: ''
+            install-musl-tools: false
+            container: quay.io/pypa/manylinux2014_x86_64
           - runs-on: ubuntu-latest
             deprecated-platform: linux_aarch64
             os: linux

.github/workflows/test_and_package_development.yml

@@ -22,14 +22,14 @@ on:
 jobs:
   release:
     name: Build, Upload, Release ABQ
-    # We only need x86-64 statically-linked binaries for development testing.
+    # We only need x86-64 many-glibc binaries for development testing.
     runs-on: ubuntu-latest
+    container: quay.io/pypa/manylinux2014_x86_64
     outputs:
       abq_version: ${{ steps.abq_version.outputs.abq_version }}
     env:
       AWS_DEFAULT_REGION: us-east-2
       AWS_DEFAULT_OUTPUT: json
-      RUST_TARGET: x86_64-unknown-linux-musl
       ABQ_DEVELOPMENT_BUILD: 1
       ABQ_NO_REBUILD: true
     steps:
@@ -50,13 +50,13 @@ jobs:
       - name: re-fetch overwritten tag
         run: git fetch --tags --force
 
+      - name: Install build + deploy deps
+        run: |
+          yum install wget sudo jq -y
+
       - name: check for typos
         uses: crate-ci/typos@master
 
-      - name: Install musl libc tools
-        run: |
-          sudo apt-get install -y musl-tools
-
       - name: Restore cargo cache
         uses: actions/cache@v3.3.1
         env:
@@ -68,14 +68,13 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: cargo-release-unstable-1-linux_x86-64-musl-${{ hashFiles('rust-toolchain.toml')}}-${{ hashFiles('Cargo.lock') }}
+          key: cargo-release-unstable-1-linux_x86-64-manylinux-${{ hashFiles('rust-toolchain.toml')}}-${{ hashFiles('Cargo.lock') }}
           restore-keys: |
-            cargo-release-unstable-1-linux_x86-64-musl-${{ hashFiles('rust-toolchain.toml')}}-
+            cargo-release-unstable-1-linux_x86-64-manylinux-${{ hashFiles('rust-toolchain.toml')}}-
       - name: Install Rust toolchain
         uses: rwx-research/rust-toolchain@abq
         with:
           toolchain: 1.65.0
-          target: ${{ env.RUST_TARGET }}
           components: clippy, rustfmt
 
       - name: Check formatting
@@ -88,7 +87,7 @@ jobs:
         uses: actions-rs/cargo@v1.0.3
         with:
           command: clippy
-          args: --workspace --tests --profile=release-unstable --all-features --target=${{ env.RUST_TARGET }} -- --deny warnings
+          args: --workspace --tests --profile=release-unstable --all-features -- --deny warnings
 
       - name: Lint github workflows
         run: |
@@ -112,13 +111,13 @@ jobs:
 
       - name: Build release with tests
         run: |
-          cargo build --tests --bins --profile=release-unstable --all-features --target="$RUST_TARGET"
+          cargo build --tests --bins --profile=release-unstable --all-features
 
       - name: Run tests with cargo
         timeout-minutes: 4
         if: github.run_attempt == 1
         run: |
-          cargo test --profile=release-unstable --all-features --target="$RUST_TARGET"
+          cargo test --profile=release-unstable --all-features
         env:
           CI: false
 
@@ -127,10 +126,16 @@ jobs:
         if: github.run_attempt > 1
         run: |
           export ABQ_DEBUG_CLI_TESTS_FOR_CI=1
-          cargo test --profile=release-unstable --all-features --target="$RUST_TARGET" -- --nocapture
+          cargo test --profile=release-unstable --all-features -- --nocapture
         env:
           CI: false
 
+      - name: Install aws CLI
+        run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+
       - name: Prepare release environment
         run: |
           ABQ_VERSION="$(cat build_artifact/abq_version.txt)"
@@ -154,8 +159,8 @@ jobs:
         id: prepare_release
         run: |
           mkdir "${RELEASE_DIR}"
-          cp "target/$RUST_TARGET/release-unstable/abq" "${RELEASE_DIR}"
-          cp "target/$RUST_TARGET/release-unstable/abq_tester_harness" "${RELEASE_DIR}"
+          cp "target/release-unstable/abq" "${RELEASE_DIR}"
+          cp "target/release-unstable/abq_tester_harness" "${RELEASE_DIR}"
           tar -czvf "${RELEASE_ARCHIVE}" "${RELEASE_DIR}"
 
       - name: Push release to staging S3

crates/abq_cli/Cargo.toml

@@ -43,9 +43,6 @@ tracing-appender.workspace = true
 
 indoc.workspace = true
 
-[target.x86_64-unknown-linux-musl.dependencies]
-mimalloc = "0.1.34"
-
 [dev-dependencies]
 abq_utils = { path = "../abq_utils", features = ["expose-native-protocols"] }
 abq_test_utils = { path = "../abq_test_support/abq_test_utils" }

crates/abq_cli/src/main.rs

@@ -39,10 +39,6 @@ use crate::{
     reporting::StdoutPreferences,
 };
 
-#[cfg(all(target_arch = "x86_64", target_env = "musl"))]
-#[global_allocator]
-static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
 fn main() -> anyhow::Result<()> {
     let exit_code = abq_main()?;
     std::process::exit(exit_code.get());

crates/abq_cli/tests/cli.rs

@@ -384,11 +384,11 @@ macro_rules! setup_queue {
 
 test_all_network_config_options! {
     #[cfg(feature = "test-abq-jest")]
-    yarn_jest_auto_workers_without_failure |name, conf: CSConfigOptions| {
-        // abq test --reporter dot (--token ...)? -- yarn jest
+    npm_jest_auto_workers_without_failure |name, conf: CSConfigOptions| {
+        // abq test --reporter dot (--token ...)? -- npm test
         let args = &["test", "--reporter", "dot", "-n", "cpu-cores", "--color=never"];
         let mut args = conf.extend_args_for_in_band_client(args);
-        args.extend(["--", "yarn", "jest"]);
+        args.extend(["--", "npm", "test"]);
         let CmdOutput {
             stdout,
             stderr: _,
@@ -405,11 +405,11 @@ test_all_network_config_options! {
 
 test_all_network_config_options! {
     #[cfg(feature = "test-abq-jest")]
-    yarn_jest_auto_workers_without_failure_worker_0 |name, conf: CSConfigOptions| {
-        // abq test --worker 0 --reporter dot (--token ...)? -- yarn jest
+    npm_jest_auto_workers_without_failure_worker_0 |name, conf: CSConfigOptions| {
+        // abq test --worker 0 --reporter dot (--token ...)? -- npm test
         let args = &["test", "--worker", "0", "--reporter", "dot", "-n", "cpu-cores", "--color=never"];
         let mut args = conf.extend_args_for_in_band_client(args);
-        args.extend(["--", "yarn", "jest"]);
+        args.extend(["--", "npm", "test"]);
         let CmdOutput {
             stdout,
             stderr: _,
@@ -506,7 +506,7 @@ fn assert_sum_of_run_test_retries<'a>(outputs: impl IntoIterator<Item = &'a str>
 
 test_all_network_config_options! {
     #[cfg(feature = "test-abq-jest")]
-    yarn_jest_separate_queue_numbered_workers_test_without_failure |name, conf: CSConfigOptions| {
+    npm_jest_separate_queue_numbered_workers_test_without_failure |name, conf: CSConfigOptions| {
         let (queue_proc, queue_addr) = setup_queue!(name, conf);
 
         let run_id = RunId::unique().to_string();
@@ -515,7 +515,7 @@ test_all_network_config_options! {
 
         let working_dir = npm_jest_project_path.display();
 
-        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest
+        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm test
         let test_args = |worker: usize| {
             let args = vec![
                 format!("test"),
@@ -528,7 +528,7 @@ test_all_network_config_options! {
                 format!("--color=never"),
             ];
             let mut args = conf.extend_args_for_client(args);
-            args.extend([s!("--"), s!("yarn"), s!("jest")]);
+            args.extend([s!("--"), s!("npm"), s!("test")]);
             args
         };
 
@@ -579,7 +579,7 @@ test_all_network_config_options! {
 
 test_all_network_config_options! {
     #[cfg(feature = "test-abq-jest")]
-    yarn_jest_timeout_run_workers |name, conf: CSConfigOptions| {
+    npm_jest_timeout_run_workers |name, conf: CSConfigOptions| {
         let (queue_proc, queue_addr) = setup_queue!(name, conf);
 
         let run_id = RunId::unique().to_string();
@@ -588,7 +588,7 @@ test_all_network_config_options! {
 
         let working_dir = npm_jest_project_path.display();
 
-        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest
+        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm test
         let test_args = |worker: usize| {
             let args = vec![
                 format!("test"),
@@ -601,7 +601,7 @@ test_all_network_config_options! {
                 format!("--inactivity-timeout-seconds=0"),
             ];
             let mut args = conf.extend_args_for_client(args);
-            args.extend([s!("--"), s!("yarn"), s!("jest")]);
+            args.extend([s!("--"), s!("npm"), s!("test")]);
             args
         };
 
@@ -619,8 +619,8 @@ test_all_network_config_options! {
 
 test_all_network_config_options! {
     #[cfg(feature = "test-abq-jest")]
-    yarn_jest_auto_workers_with_failing_tests |name, conf: CSConfigOptions| {
-        // abq test --reporter dot (--token ...)? -- yarn jest
+    npm_jest_auto_workers_with_failing_tests |name, conf: CSConfigOptions| {
+        // abq test --reporter dot (--token ...)? -- npm test
         let test_args = &[
             "test",
             "--reporter",
@@ -631,7 +631,7 @@ test_all_network_config_options! {
             "--worker=0",
         ];
         let mut test_args = conf.extend_args_for_in_band_client(test_args);
-        test_args.extend(["--", "yarn", "jest"]);
+        test_args.extend(["--", "npm", "test"]);
         let CmdOutput {
             stdout,
             stderr: _,
@@ -652,15 +652,15 @@ test_all_network_config_options! {
 
 test_all_network_config_options! {
     #[cfg(feature = "test-abq-jest")]
-    yarn_jest_separate_queue_workers_with_failing_tests |name, conf: CSConfigOptions| {
+    npm_jest_separate_queue_workers_with_failing_tests |name, conf: CSConfigOptions| {
         let (queue_proc, queue_addr) = setup_queue!(name, conf);
 
         let run_id = RunId::unique().to_string();
 
         let npm_jest_project_path = testdata_project("jest/npm-jest-project-with-failures");
         let working_dir = npm_jest_project_path.display();
 
-        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest
+        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm test
         let test_args = |worker: usize| {
             let args = vec![
                 format!("test"),
@@ -673,7 +673,7 @@ test_all_network_config_options! {
                 format!("--color=never"),
             ];
             let mut args = conf.extend_args_for_client(args);
-            args.extend([s!("--"), s!("yarn"), s!("jest")]);
+            args.extend([s!("--"), s!("npm"), s!("test")]);
             args
         };
         let test1_proc = Abq::new(name.to_string() + "_test1").args(test_args(1)).spawn();
@@ -2245,7 +2245,7 @@ test_all_network_config_options! {
 
         let simulation = [pack_msgs_to_disk(simulation(true)), pack_msgs_to_disk(simulation(false))];
 
-        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- yarn jest
+        // abq test --worker N --reporter dot --queue-addr ... --working-dir ... --run-id ... (--token ...)? -- npm jest
         let test_args = |worker: usize| {
             let simulator = native_runner_simulation_bin();
             let simulation = &simulation[worker];

crates/abq_test_support/abq_test_utils/src/lib.rs

@@ -27,12 +27,13 @@ pub mod color_writer;
 pub const WORKSPACE: &str = env!("ABQ_WORKSPACE_DIR");
 
 pub fn artifacts_dir() -> PathBuf {
-    let path = if cfg!(all(target_arch = "x86_64", target_env = "musl")) {
-        // GHA is using a musl target
+    let path = if cfg!(all(target_arch = "x86_64")) {
+        // GHA x86 linux.
         if cfg!(debug_assertions) {
-            "target/x86_64-unknown-linux-musl/release-unstable"
+            // GHA x86 with debug assertions targets the release-unstable profile.
+            "target/release-unstable"
         } else {
-            "target/x86_64-unknown-linux-musl/release"
+            "target/release"
         }
     } else if cfg!(debug_assertions) {
         "target/debug"