From f75fda17f99bf5723e4bcb94482c94b30b5b1c96 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 14 May 2026 17:37:25 -0700 Subject: [PATCH] Revert "perf(build): speed up local CLI rebuilds (#1387)" This reverts commit 668c712b6345cccc1b976c52c59a24b5dcf6654f. --- Cargo.lock | 20 +- Cargo.toml | 8 +- THIRD-PARTY-NOTICES | 210 +++++++++++++++++++ crates/openshell-cli/src/run.rs | 26 +-- crates/openshell-core/Cargo.toml | 2 +- crates/openshell-core/build.rs | 55 ++--- deploy/docker/Dockerfile.cli-macos | 1 - deploy/docker/Dockerfile.driver-vm-macos | 1 - deploy/docker/Dockerfile.gateway-macos | 1 - deploy/docker/Dockerfile.python-wheels | 1 - deploy/docker/Dockerfile.python-wheels-macos | 1 - scripts/bin/openshell | 6 +- 12 files changed, 254 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c55118a2a..29919124a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + [[package]] name = "aws-lc-rs" version = "1.16.3" @@ -3386,8 +3395,8 @@ dependencies = [ "ipnet", "miette", "prost", - "prost-build", "prost-types", + "protobuf-src", "serde", "serde_json", "tempfile", @@ -4225,6 +4234,15 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf-src" +version = "1.1.0+21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +dependencies = [ + "autotools", +] + [[package]] name = "quanta" version = "0.12.6" diff --git a/Cargo.toml b/Cargo.toml index e093c8aa0..195544431 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ tokio = { version = "1.43", features = ["full"] } # gRPC/Protobuf tonic = "0.12" tonic-build = "0.12" -prost-build = "0.13" prost = "0.13" prost-types = "0.13" @@ -98,6 +97,7 @@ futures = "0.3" bytes = "1" pin-project-lite = "0.2" tokio-stream = "0.1" +protobuf-src = "1.1.0" url = "2" # Database @@ -149,9 +149,3 @@ strip = true [profile.dev] # Faster compile times for dev builds debug = 1 - -# Compile build scripts and proc-macros with optimizations so they run faster. -# This speeds up tonic-build, clap derive, serde derive, etc. at the cost of a -# one-time slower compilation of those crates. -[profile.dev.build-override] -opt-level = 2 diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES index ae0873486..41eda0e0a 100644 --- a/THIRD-PARTY-NOTICES +++ b/THIRD-PARTY-NOTICES @@ -870,6 +870,216 @@ Used by: See the License for the specific language governing permissions and limitations under the License. +================================================================================ +License: Apache-2.0 +-------------------------------------------------------------------------------- + +Used by: + - protobuf-src 1.1.0+21.5 (https://github.com/MaterializeInc/rust-protobuf-native) + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2007] Neal Norwitz + Portions Copyright [2007] Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. ================================================================================ License: Apache-2.0 diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 81abd21c7..bc29e6b5b 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -14,9 +14,7 @@ use futures::StreamExt; use http_body_util::Full; use hyper::{Request, StatusCode}; use hyper_rustls::HttpsConnectorBuilder; -use hyper_util::{ - client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor, -}; +use hyper_util::{client::legacy::Client, rt::TokioExecutor}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use miette::{IntoDiagnostic, Result, WrapErr, miette}; use openshell_bootstrap::{ @@ -1273,14 +1271,6 @@ async fn http_health_check(server: &str, tls: &TlsOptions) -> Result> = - Client::builder(TokioExecutor::new()).build_http(); - let req = health_check_request(uri, tls)?; - let resp = client.request(req).await.into_diagnostic()?; - return Ok(Some(resp.status())); - } - let https = if tls.gateway_insecure && scheme.eq_ignore_ascii_case("https") { let insecure_config = build_insecure_rustls_config()?; HttpsConnectorBuilder::new() @@ -1288,7 +1278,7 @@ async fn http_health_check(server: &str, tls: &TlsOptions) -> Result Result> = Client::builder(TokioExecutor::new()).build(https); - let req = health_check_request(uri, tls)?; - let resp = client.request(req).await.into_diagnostic()?; - Ok(Some(resp.status())) -} - -fn health_check_request(uri: hyper::Uri, tls: &TlsOptions) -> Result>> { let mut req_builder = Request::builder().method("GET").uri(uri); // Inject edge authentication headers when an edge token is configured. if let Some(ref token) = tls.edge_token { @@ -1318,7 +1302,11 @@ fn health_check_request(uri: hyper::Uri, tls: &TlsOptions) -> Result bool { diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index fcab2e286..b03fb1494 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -29,7 +29,7 @@ dev-settings = [] [build-dependencies] tonic-build = { workspace = true } -prost-build = { workspace = true } +protobuf-src = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 8360f67d1..7613c8754 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -12,12 +12,9 @@ fn main() -> Result<(), Box> { // builds where .git is absent, this silently does nothing and the binary // falls back to CARGO_PKG_VERSION (which is already sed-patched by the // build pipeline). - // - // We intentionally do NOT set `rerun-if-changed` for .git/HEAD or - // .git/refs/tags. Watching those paths re-triggers protobuf codegen and - // cascades a rebuild of every downstream crate on every commit. The git - // version is refreshed whenever proto files change or the cargo cache is - // cleared, which is sufficient for development. + println!("cargo:rerun-if-changed=../../.git/HEAD"); + println!("cargo:rerun-if-changed=../../.git/refs/tags"); + if let Some(version) = git_version() { println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}"); } @@ -25,6 +22,16 @@ fn main() -> Result<(), Box> { // --- Protobuf compilation --- // Re-run when anything under proto/ changes (including newly added .proto files). println!("cargo:rerun-if-changed={PROTO_REL}"); + // Use bundled protoc from protobuf-src. The system protoc (from apt-get) + // does not bundle the well-known type includes (google/protobuf/struct.proto + // etc.), so we must use protobuf-src which ships both the binary and the + // include tree. + // SAFETY: This is run at build time in a single-threaded build script context. + // No other threads are reading environment variables concurrently. + #[allow(unsafe_code)] + unsafe { + env::set_var("PROTOC", protobuf_src::protoc()); + } let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let proto_root = manifest_dir.join(PROTO_REL); @@ -33,47 +40,15 @@ fn main() -> Result<(), Box> { collect_proto_files(&proto_root, &mut proto_files)?; proto_files.sort(); - // Requires `protoc`. Local and CI builds get it from mise; Docker build - // images install protobuf-compiler. Those protoc distributions also - // provide the well-known type includes used by imports such as - // google/protobuf/struct.proto. - let mut prost_config = prost_build::Config::new(); - if let Some(protoc) = resolve_protoc_from_mise() { - prost_config.protoc_executable(protoc); - } + // Configure tonic-build tonic_build::configure() .build_server(true) .build_client(true) - .compile_protos_with_config(prost_config, &proto_files, &[proto_root.as_path()])?; + .compile_protos(&proto_files, &[proto_root.as_path()])?; Ok(()) } -fn resolve_protoc_from_mise() -> Option { - if env::var_os("PROTOC").is_some() || command_exists("protoc") { - return None; - } - - let output = std::process::Command::new("mise") - .args(["where", "protoc"]) - .output() - .ok()?; - if !output.status.success() { - return None; - } - - let root = String::from_utf8(output.stdout).ok()?; - let protoc = PathBuf::from(root.trim()).join("bin").join("protoc"); - protoc.is_file().then_some(protoc) -} - -fn command_exists(command: &str) -> bool { - std::process::Command::new(command) - .arg("--version") - .output() - .is_ok_and(|output| output.status.success()) -} - fn collect_proto_files(dir: &Path, out: &mut Vec) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let path = entry?.path(); diff --git a/deploy/docker/Dockerfile.cli-macos b/deploy/docker/Dockerfile.cli-macos index c187cb3fd..7dce3a63d 100644 --- a/deploy/docker/Dockerfile.cli-macos +++ b/deploy/docker/Dockerfile.cli-macos @@ -36,7 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libclang-dev \ pkg-config \ - protobuf-compiler \ && rm -rf /var/lib/apt/lists/* RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 diff --git a/deploy/docker/Dockerfile.driver-vm-macos b/deploy/docker/Dockerfile.driver-vm-macos index 0808915a2..58317a52d 100644 --- a/deploy/docker/Dockerfile.driver-vm-macos +++ b/deploy/docker/Dockerfile.driver-vm-macos @@ -39,7 +39,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ curl \ pkg-config \ - protobuf-compiler \ && rm -rf /var/lib/apt/lists/* RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 diff --git a/deploy/docker/Dockerfile.gateway-macos b/deploy/docker/Dockerfile.gateway-macos index 15549454e..27d2ffbba 100644 --- a/deploy/docker/Dockerfile.gateway-macos +++ b/deploy/docker/Dockerfile.gateway-macos @@ -30,7 +30,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libclang-dev \ pkg-config \ - protobuf-compiler \ && rm -rf /var/lib/apt/lists/* RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 diff --git a/deploy/docker/Dockerfile.python-wheels b/deploy/docker/Dockerfile.python-wheels index fa6beaf6b..e93bf8f22 100644 --- a/deploy/docker/Dockerfile.python-wheels +++ b/deploy/docker/Dockerfile.python-wheels @@ -18,7 +18,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libc6-dev \ libclang-dev \ pkg-config \ - protobuf-compiler \ libssl-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/deploy/docker/Dockerfile.python-wheels-macos b/deploy/docker/Dockerfile.python-wheels-macos index 65a307718..45194b069 100644 --- a/deploy/docker/Dockerfile.python-wheels-macos +++ b/deploy/docker/Dockerfile.python-wheels-macos @@ -31,7 +31,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libclang-dev \ libssl-dev \ pkg-config \ - protobuf-compiler \ && rm -rf /var/lib/apt/lists/* # aws-lc-sys probes with --target=arm64-apple-macosx and clang then looks for diff --git a/scripts/bin/openshell b/scripts/bin/openshell index 52f89ab0d..23000bc31 100755 --- a/scripts/bin/openshell +++ b/scripts/bin/openshell @@ -115,11 +115,7 @@ if [[ "$needs_build" == "1" ]]; then build_args+=(--features bundled-z3) fi fi - if [[ "${OPENSHELL_CLI_USE_SCCACHE:-}" == "1" ]]; then - cargo build "${build_args[@]}" - else - env -u RUSTC_WRAPPER cargo build "${build_args[@]}" - fi + cargo build "${build_args[@]}" # Persist state after successful build mkdir -p "$(dirname "$STATE_FILE")" cd "$PROJECT_ROOT"