From c8c2710c39a639edde02a0863c9632e01ba66084 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 14 May 2026 10:36:11 -0700 Subject: [PATCH] perf(build): speed up local CLI rebuilds --- Cargo.lock | 20 +- Cargo.toml | 8 +- THIRD-PARTY-NOTICES | 210 ------------------- crates/openshell-cli/src/run.rs | 26 ++- crates/openshell-core/Cargo.toml | 2 +- crates/openshell-core/build.rs | 55 +++-- deploy/docker/Dockerfile.cli-macos | 1 + deploy/docker/Dockerfile.driver-vm-macos | 1 + deploy/docker/Dockerfile.gateway-macos | 1 + deploy/docker/Dockerfile.python-wheels | 1 + deploy/docker/Dockerfile.python-wheels-macos | 1 + scripts/bin/openshell | 6 +- 12 files changed, 78 insertions(+), 254 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29919124a..c55118a2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,15 +232,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "autotools" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" -dependencies = [ - "cc", -] - [[package]] name = "aws-lc-rs" version = "1.16.3" @@ -3395,8 +3386,8 @@ dependencies = [ "ipnet", "miette", "prost", + "prost-build", "prost-types", - "protobuf-src", "serde", "serde_json", "tempfile", @@ -4234,15 +4225,6 @@ dependencies = [ "prost", ] -[[package]] -name = "protobuf-src" -version = "1.1.0+21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" -dependencies = [ - "autotools", -] - [[package]] name = "quanta" version = "0.12.6" diff --git a/Cargo.toml b/Cargo.toml index 195544431..e093c8aa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ tokio = { version = "1.43", features = ["full"] } # gRPC/Protobuf tonic = "0.12" tonic-build = "0.12" +prost-build = "0.13" prost = "0.13" prost-types = "0.13" @@ -97,7 +98,6 @@ futures = "0.3" bytes = "1" pin-project-lite = "0.2" tokio-stream = "0.1" -protobuf-src = "1.1.0" url = "2" # Database @@ -149,3 +149,9 @@ strip = true [profile.dev] # Faster compile times for dev builds debug = 1 + +# Compile build scripts and proc-macros with optimizations so they run faster. +# This speeds up tonic-build, clap derive, serde derive, etc. at the cost of a +# one-time slower compilation of those crates. +[profile.dev.build-override] +opt-level = 2 diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES index 41eda0e0a..ae0873486 100644 --- a/THIRD-PARTY-NOTICES +++ b/THIRD-PARTY-NOTICES @@ -870,216 +870,6 @@ Used by: See the License for the specific language governing permissions and limitations under the License. -================================================================================ -License: Apache-2.0 --------------------------------------------------------------------------------- - -Used by: - - protobuf-src 1.1.0+21.5 (https://github.com/MaterializeInc/rust-protobuf-native) - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [2007] Neal Norwitz - Portions Copyright [2007] Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ================================================================================ License: Apache-2.0 diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index bc29e6b5b..81abd21c7 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -14,7 +14,9 @@ use futures::StreamExt; use http_body_util::Full; use hyper::{Request, StatusCode}; use hyper_rustls::HttpsConnectorBuilder; -use hyper_util::{client::legacy::Client, rt::TokioExecutor}; +use hyper_util::{ + client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor, +}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use miette::{IntoDiagnostic, Result, WrapErr, miette}; use openshell_bootstrap::{ @@ -1271,6 +1273,14 @@ async fn http_health_check(server: &str, tls: &TlsOptions) -> Result> = + Client::builder(TokioExecutor::new()).build_http(); + let req = health_check_request(uri, tls)?; + let resp = client.request(req).await.into_diagnostic()?; + return Ok(Some(resp.status())); + } + let https = if tls.gateway_insecure && scheme.eq_ignore_ascii_case("https") { let insecure_config = build_insecure_rustls_config()?; HttpsConnectorBuilder::new() @@ -1278,7 +1288,7 @@ async fn http_health_check(server: &str, tls: &TlsOptions) -> Result Result> = Client::builder(TokioExecutor::new()).build(https); + let req = health_check_request(uri, tls)?; + let resp = client.request(req).await.into_diagnostic()?; + Ok(Some(resp.status())) +} + +fn health_check_request(uri: hyper::Uri, tls: &TlsOptions) -> Result>> { let mut req_builder = Request::builder().method("GET").uri(uri); // Inject edge authentication headers when an edge token is configured. if let Some(ref token) = tls.edge_token { @@ -1302,11 +1318,7 @@ async fn http_health_check(server: &str, tls: &TlsOptions) -> Result bool { diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index b03fb1494..fcab2e286 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -29,7 +29,7 @@ dev-settings = [] [build-dependencies] tonic-build = { workspace = true } -protobuf-src = { workspace = true } +prost-build = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 7613c8754..8360f67d1 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -12,9 +12,12 @@ fn main() -> Result<(), Box> { // builds where .git is absent, this silently does nothing and the binary // falls back to CARGO_PKG_VERSION (which is already sed-patched by the // build pipeline). - println!("cargo:rerun-if-changed=../../.git/HEAD"); - println!("cargo:rerun-if-changed=../../.git/refs/tags"); - + // + // We intentionally do NOT set `rerun-if-changed` for .git/HEAD or + // .git/refs/tags. Watching those paths re-triggers protobuf codegen and + // cascades a rebuild of every downstream crate on every commit. The git + // version is refreshed whenever proto files change or the cargo cache is + // cleared, which is sufficient for development. if let Some(version) = git_version() { println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}"); } @@ -22,16 +25,6 @@ fn main() -> Result<(), Box> { // --- Protobuf compilation --- // Re-run when anything under proto/ changes (including newly added .proto files). println!("cargo:rerun-if-changed={PROTO_REL}"); - // Use bundled protoc from protobuf-src. The system protoc (from apt-get) - // does not bundle the well-known type includes (google/protobuf/struct.proto - // etc.), so we must use protobuf-src which ships both the binary and the - // include tree. - // SAFETY: This is run at build time in a single-threaded build script context. - // No other threads are reading environment variables concurrently. - #[allow(unsafe_code)] - unsafe { - env::set_var("PROTOC", protobuf_src::protoc()); - } let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let proto_root = manifest_dir.join(PROTO_REL); @@ -40,15 +33,47 @@ fn main() -> Result<(), Box> { collect_proto_files(&proto_root, &mut proto_files)?; proto_files.sort(); - // Configure tonic-build + // Requires `protoc`. Local and CI builds get it from mise; Docker build + // images install protobuf-compiler. Those protoc distributions also + // provide the well-known type includes used by imports such as + // google/protobuf/struct.proto. + let mut prost_config = prost_build::Config::new(); + if let Some(protoc) = resolve_protoc_from_mise() { + prost_config.protoc_executable(protoc); + } tonic_build::configure() .build_server(true) .build_client(true) - .compile_protos(&proto_files, &[proto_root.as_path()])?; + .compile_protos_with_config(prost_config, &proto_files, &[proto_root.as_path()])?; Ok(()) } +fn resolve_protoc_from_mise() -> Option { + if env::var_os("PROTOC").is_some() || command_exists("protoc") { + return None; + } + + let output = std::process::Command::new("mise") + .args(["where", "protoc"]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + + let root = String::from_utf8(output.stdout).ok()?; + let protoc = PathBuf::from(root.trim()).join("bin").join("protoc"); + protoc.is_file().then_some(protoc) +} + +fn command_exists(command: &str) -> bool { + std::process::Command::new(command) + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) +} + fn collect_proto_files(dir: &Path, out: &mut Vec) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let path = entry?.path(); diff --git a/deploy/docker/Dockerfile.cli-macos b/deploy/docker/Dockerfile.cli-macos index 7dce3a63d..c187cb3fd 100644 --- a/deploy/docker/Dockerfile.cli-macos +++ b/deploy/docker/Dockerfile.cli-macos @@ -36,6 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libclang-dev \ pkg-config \ + protobuf-compiler \ && rm -rf /var/lib/apt/lists/* RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 diff --git a/deploy/docker/Dockerfile.driver-vm-macos b/deploy/docker/Dockerfile.driver-vm-macos index 58317a52d..0808915a2 100644 --- a/deploy/docker/Dockerfile.driver-vm-macos +++ b/deploy/docker/Dockerfile.driver-vm-macos @@ -39,6 +39,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ curl \ pkg-config \ + protobuf-compiler \ && rm -rf /var/lib/apt/lists/* RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 diff --git a/deploy/docker/Dockerfile.gateway-macos b/deploy/docker/Dockerfile.gateway-macos index 27d2ffbba..15549454e 100644 --- a/deploy/docker/Dockerfile.gateway-macos +++ b/deploy/docker/Dockerfile.gateway-macos @@ -30,6 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libclang-dev \ pkg-config \ + protobuf-compiler \ && rm -rf /var/lib/apt/lists/* RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.95.0 diff --git a/deploy/docker/Dockerfile.python-wheels b/deploy/docker/Dockerfile.python-wheels index e93bf8f22..fa6beaf6b 100644 --- a/deploy/docker/Dockerfile.python-wheels +++ b/deploy/docker/Dockerfile.python-wheels @@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libc6-dev \ libclang-dev \ pkg-config \ + protobuf-compiler \ libssl-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/deploy/docker/Dockerfile.python-wheels-macos b/deploy/docker/Dockerfile.python-wheels-macos index 45194b069..65a307718 100644 --- a/deploy/docker/Dockerfile.python-wheels-macos +++ b/deploy/docker/Dockerfile.python-wheels-macos @@ -31,6 +31,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libclang-dev \ libssl-dev \ pkg-config \ + protobuf-compiler \ && rm -rf /var/lib/apt/lists/* # aws-lc-sys probes with --target=arm64-apple-macosx and clang then looks for diff --git a/scripts/bin/openshell b/scripts/bin/openshell index 23000bc31..52f89ab0d 100755 --- a/scripts/bin/openshell +++ b/scripts/bin/openshell @@ -115,7 +115,11 @@ if [[ "$needs_build" == "1" ]]; then build_args+=(--features bundled-z3) fi fi - cargo build "${build_args[@]}" + if [[ "${OPENSHELL_CLI_USE_SCCACHE:-}" == "1" ]]; then + cargo build "${build_args[@]}" + else + env -u RUSTC_WRAPPER cargo build "${build_args[@]}" + fi # Persist state after successful build mkdir -p "$(dirname "$STATE_FILE")" cd "$PROJECT_ROOT"