From 8055b09acd4fd2ddc272a6616f597bb669fda507 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 24 Jul 2026 15:32:38 -0700 Subject: [PATCH 1/2] fix(core): resolve git metadata in worktrees Signed-off-by: Matthew Grossman --- crates/openshell-core/build.rs | 9 +- crates/openshell-core/build_support/git.rs | 45 +++++++++ .../openshell-core/tests/git_watch_paths.rs | 92 +++++++++++++++++++ 3 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 crates/openshell-core/build_support/git.rs create mode 100644 crates/openshell-core/tests/git_watch_paths.rs diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 7955772a67..7e503b109a 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -4,16 +4,20 @@ use std::env; use std::path::{Path, PathBuf}; +#[path = "build_support/git.rs"] +mod git; + const PROTO_REL: &str = "../../proto"; fn main() -> Result<(), Box> { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + // --- Git-derived version --- // Compute a version from `git describe` for local builds. In Docker/CI // builds where .git is absent, this silently does nothing and the binary // falls back to CARGO_PKG_VERSION (which is already sed-patched by the // build pipeline). - println!("cargo:rerun-if-changed=../../.git/HEAD"); - println!("cargo:rerun-if-changed=../../.git/refs/tags"); + git::emit_rerun_if_changed(&manifest_dir); if let Some(version) = git_version() { println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}"); @@ -33,7 +37,6 @@ fn main() -> Result<(), Box> { env::set_var("PROTOC", protobuf_src::protoc()); } - let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let proto_root = manifest_dir.join(PROTO_REL); let mut proto_files = Vec::new(); diff --git a/crates/openshell-core/build_support/git.rs b/crates/openshell-core/build_support/git.rs new file mode 100644 index 0000000000..99c877083f --- /dev/null +++ b/crates/openshell-core/build_support/git.rs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const WATCH_PATHS: [&str; 2] = ["HEAD", "refs/tags"]; + +#[cfg(not(test))] +pub fn emit_rerun_if_changed(manifest_dir: &Path) { + for path in watch_paths(manifest_dir) { + println!("cargo:rerun-if-changed={}", path.display()); + } +} + +pub fn watch_paths(manifest_dir: &Path) -> Vec { + WATCH_PATHS + .into_iter() + .filter_map(|path| resolve_existing_git_path(manifest_dir, path)) + .collect::>() + .into_iter() + .collect() +} + +fn resolve_existing_git_path(manifest_dir: &Path, path: &str) -> Option { + let output = Command::new("git") + .args(["rev-parse", "--git-path", path]) + .current_dir(manifest_dir) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + let path = PathBuf::from(String::from_utf8(output.stdout).ok()?.trim()); + let path = if path.is_absolute() { + path + } else { + manifest_dir.join(path) + }; + + path.exists().then_some(path) +} diff --git a/crates/openshell-core/tests/git_watch_paths.rs b/crates/openshell-core/tests/git_watch_paths.rs new file mode 100644 index 0000000000..b2769e4333 --- /dev/null +++ b/crates/openshell-core/tests/git_watch_paths.rs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[path = "../build_support/git.rs"] +mod git; + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn run_git(repo: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout) + .expect("git output is UTF-8") + .trim() + .to_owned() +} + +fn init_repository() -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().expect("create temp directory"); + let repo = temp.path().join("repo"); + fs::create_dir(&repo).expect("create repository directory"); + + run_git(&repo, &["init", "--quiet"]); + run_git(&repo, &["config", "user.name", "OpenShell Test"]); + run_git( + &repo, + &["config", "user.email", "openshell@example.invalid"], + ); + fs::write(repo.join("README.md"), "test\n").expect("write tracked file"); + run_git(&repo, &["add", "README.md"]); + run_git(&repo, &["commit", "--quiet", "-m", "initial"]); + run_git(&repo, &["tag", "v0.0.1"]); + + (temp, repo) +} + +fn git_path(repo: &Path, path: &str) -> PathBuf { + let path = PathBuf::from(run_git(repo, &["rev-parse", "--git-path", path])); + if path.is_absolute() { + path + } else { + repo.join(path) + } +} + +#[test] +fn resolves_metadata_paths_in_normal_checkout() { + let (_temp, repo) = init_repository(); + + let paths = git::watch_paths(&repo); + + assert!(paths.contains(&git_path(&repo, "HEAD"))); + assert!(paths.contains(&git_path(&repo, "refs/tags"))); + assert!(paths.iter().all(|path| path.exists())); +} + +#[test] +fn resolves_metadata_paths_in_linked_worktree() { + let (temp, repo) = init_repository(); + let worktree = temp.path().join("linked"); + let worktree_arg = worktree.to_str().expect("worktree path is UTF-8"); + run_git( + &repo, + &["worktree", "add", "--quiet", "-b", "linked", worktree_arg], + ); + + let paths = git::watch_paths(&worktree); + let head = git_path(&worktree, "HEAD"); + + assert!(paths.contains(&head)); + assert_ne!(head, worktree.join(".git/HEAD")); + assert!(paths.contains(&git_path(&worktree, "refs/tags"))); + assert!(paths.iter().all(|path| path.exists())); +} + +#[test] +fn ignores_source_tree_without_git_metadata() { + let temp = tempfile::tempdir().expect("create temp directory"); + + assert!(git::watch_paths(temp.path()).is_empty()); +} From 5f23b3a1245f095925a0aa56cd6644d54f8977bb Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 27 Jul 2026 14:26:00 -0700 Subject: [PATCH 2/2] perf(build): improve worktree cache reuse Signed-off-by: Matthew Grossman --- CONTRIBUTING.md | 28 +++++++++++-------- crates/openshell-core/build.rs | 5 ++-- crates/openshell-core/build_support/git.rs | 28 +++++++++++++++++-- .../openshell-core/tests/git_watch_paths.rs | 25 +++++++++++++++++ mise.toml | 7 +++-- tasks/rust.toml | 17 +++++++++++ 6 files changed, 91 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0465f88f7d..c2d1d24c42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -200,23 +200,27 @@ openshell sandbox create -- codex Mise preserves an existing `SCCACHE_DIR` so each environment can choose where to store compiler cache entries. When `SCCACHE_DIR` is unset, OpenShell uses -the worktree-local `.cache/sccache` directory. To make cache entries available -to multiple worktrees on a workstation, set the variable to a user-level -directory before activating mise. For example: +the user-level `openshell/sccache` directory under the platform cache directory. +All worktrees therefore share compiler outputs while retaining separate Cargo +artifacts in each worktree's `target/` directory. + +The sccache daemon retains the environment with which it started. After changing +`SCCACHE_DIR` or other sccache settings, inspect the requested and active +configuration: + +```shell +mise run rust:cache:status +``` + +If the requested and active settings differ, refresh the daemon when no other +builds are running: ```shell -export SCCACHE_DIR="$HOME/.cache/openshell/sccache" +mise run rust:cache:refresh ``` CI can select a different directory or configure a remote sccache backend -without changing the workstation setting. Cargo output remains in each -worktree's `target/` directory. - -OpenShell does not set `SCCACHE_BASEDIRS`. Sccache loads base directories when -its machine-local daemon starts, but the correct workspace root differs for -each worktree. Cache reuse therefore depends on the compiler inputs: outputs -that embed absolute paths, including Rust dependencies in some builds, can -still miss across worktrees. +without changing the workstation defaults. ## Main Tasks diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 7e503b109a..6f93f59520 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -19,7 +19,7 @@ fn main() -> Result<(), Box> { // build pipeline). git::emit_rerun_if_changed(&manifest_dir); - if let Some(version) = git_version() { + if let Some(version) = git_version(&manifest_dir) { println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}"); } @@ -86,7 +86,7 @@ fn collect_proto_files(dir: &Path, out: &mut Vec) -> std::io::Result<() /// 3 commits past v0.0.3 → "0.0.4-dev.3+g2bf9969" /// /// Returns `None` when git is unavailable or the repo has no matching tags. -fn git_version() -> Option { +fn git_version(manifest_dir: &Path) -> Option { // Match numeric release tags only (e.g. `v0.0.29`). The bare glob `v*` // also matches non-release tags like `vm-dev` or `vm-prod`; when one of // those lands on the same commit as a release tag, `git describe` picks @@ -95,6 +95,7 @@ fn git_version() -> Option { // those development tags without losing any release tag. let output = std::process::Command::new("git") .args(["describe", "--tags", "--long", "--match", "v[0-9]*"]) + .current_dir(manifest_dir) .output() .ok()?; diff --git a/crates/openshell-core/build_support/git.rs b/crates/openshell-core/build_support/git.rs index 99c877083f..8205d6aab4 100644 --- a/crates/openshell-core/build_support/git.rs +++ b/crates/openshell-core/build_support/git.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::process::Command; -const WATCH_PATHS: [&str; 2] = ["HEAD", "refs/tags"]; +const WATCH_PATHS: [&str; 3] = ["HEAD", "refs/tags", "packed-refs"]; #[cfg(not(test))] pub fn emit_rerun_if_changed(manifest_dir: &Path) { @@ -15,14 +15,38 @@ pub fn emit_rerun_if_changed(manifest_dir: &Path) { } pub fn watch_paths(manifest_dir: &Path) -> Vec { - WATCH_PATHS + let mut git_paths = WATCH_PATHS .into_iter() + .map(str::to_owned) + .collect::>(); + + if let Some(head_ref) = symbolic_head_ref(manifest_dir) { + git_paths.push(head_ref); + } + + git_paths + .iter() .filter_map(|path| resolve_existing_git_path(manifest_dir, path)) .collect::>() .into_iter() .collect() } +fn symbolic_head_ref(manifest_dir: &Path) -> Option { + let output = Command::new("git") + .args(["symbolic-ref", "--quiet", "HEAD"]) + .current_dir(manifest_dir) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + let head_ref = String::from_utf8(output.stdout).ok()?.trim().to_owned(); + (!head_ref.is_empty()).then_some(head_ref) +} + fn resolve_existing_git_path(manifest_dir: &Path, path: &str) -> Option { let output = Command::new("git") .args(["rev-parse", "--git-path", path]) diff --git a/crates/openshell-core/tests/git_watch_paths.rs b/crates/openshell-core/tests/git_watch_paths.rs index b2769e4333..25e87f82f0 100644 --- a/crates/openshell-core/tests/git_watch_paths.rs +++ b/crates/openshell-core/tests/git_watch_paths.rs @@ -59,8 +59,10 @@ fn resolves_metadata_paths_in_normal_checkout() { let (_temp, repo) = init_repository(); let paths = git::watch_paths(&repo); + let head_ref = run_git(&repo, &["symbolic-ref", "HEAD"]); assert!(paths.contains(&git_path(&repo, "HEAD"))); + assert!(paths.contains(&git_path(&repo, &head_ref))); assert!(paths.contains(&git_path(&repo, "refs/tags"))); assert!(paths.iter().all(|path| path.exists())); } @@ -77,13 +79,36 @@ fn resolves_metadata_paths_in_linked_worktree() { let paths = git::watch_paths(&worktree); let head = git_path(&worktree, "HEAD"); + let head_ref = run_git(&worktree, &["symbolic-ref", "HEAD"]); assert!(paths.contains(&head)); assert_ne!(head, worktree.join(".git/HEAD")); + assert!(paths.contains(&git_path(&worktree, &head_ref))); assert!(paths.contains(&git_path(&worktree, "refs/tags"))); assert!(paths.iter().all(|path| path.exists())); } +#[test] +fn watches_packed_refs() { + let (_temp, repo) = init_repository(); + run_git(&repo, &["pack-refs", "--all"]); + + let paths = git::watch_paths(&repo); + + assert!(paths.contains(&git_path(&repo, "packed-refs"))); +} + +#[test] +fn detached_head_does_not_require_a_symbolic_ref() { + let (_temp, repo) = init_repository(); + run_git(&repo, &["checkout", "--quiet", "--detach"]); + + let paths = git::watch_paths(&repo); + + assert!(paths.contains(&git_path(&repo, "HEAD"))); + assert!(paths.iter().all(|path| path.exists())); +} + #[test] fn ignores_source_tree_without_git_metadata() { let temp = tempfile::tempdir().expect("create temp directory"); diff --git a/mise.toml b/mise.toml index a5d9682108..5a574b2cb8 100644 --- a/mise.toml +++ b/mise.toml @@ -53,10 +53,11 @@ _.file = [".env"] KUBECONFIG = "{{config_root}}/kubeconfig" UV_CACHE_DIR = "{{config_root}}/.cache/uv" -# Enable sccache for faster Rust builds. Preserve an SCCACHE_DIR selected by the -# caller so workstations and CI can configure their cache locations separately. +# Enable sccache for faster Rust builds. Preserve caller overrides, use one +# user-level cache for local worktrees, and retain the repository-local path +# expected by CI cache steps. RUSTC_WRAPPER = "sccache" -SCCACHE_DIR = "{{ get_env(name='SCCACHE_DIR', default=config_root ~ '/.cache/sccache') }}" +SCCACHE_DIR = "{{ get_env(name='SCCACHE_DIR', default=(config_root ~ '/.cache/sccache') if get_env(name='CI', default='') else (xdg_cache_home ~ '/openshell/sccache')) }}" # Shared build constants (overridable via environment). # DOCKER_BUILDKIT enables BuildKit for Docker; ignored by podman. diff --git a/tasks/rust.toml b/tasks/rust.toml index f035193fd8..9341b555f5 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -3,6 +3,23 @@ # Rust check, lint, and format tasks +["rust:cache:status"] +description = "Show requested and active Rust compiler cache settings" +run = ''' +echo "Requested cache directory: ${SCCACHE_DIR:-}" +echo "Requested base directories: ${SCCACHE_BASEDIRS:-}" +echo +sccache --show-stats +''' + +["rust:cache:refresh"] +description = "Restart sccache with the current worktree cache settings" +run = ''' +sccache --stop-server || true +sccache --start-server +sccache --show-stats +''' + ["rust:check"] description = "Check all Rust crates for errors" run = "cargo check --workspace"