feat: fast worktree setup via a shared frozen deps store

.gitignore

@@ -39,3 +39,7 @@ __pycache__
 
 # Local opt-in: install nargo from the matching official noir release instead of building from source.
 noir-from-release.flag
+
+# Manifests written by scripts/worktrees.sh (link-mode deps store; local-dev only).
+.deps-manifest.json
+.deps-manifest.linked

CLAUDE.md

@@ -53,6 +53,10 @@ Follow Conventional Commits: `fix:`, `feat:`, `chore:`, `refactor:`, `docs:`, `t
 When staging files, prefer `git add -u` or name specific files rather than `git add -A` or `git add .`. The aggregate flags will pick up unrelated untracked working directories (e.g. personal scratch projects at the repo root) and quietly stage them. Subagents must always name specific files in `git add` — never `-u`, `-A`, or `.` — because they lack the main conversation's context for judging which changes belong to the current task.
 </git_staging>
 
+<worktrees>
+To create a git worktree, use `scripts/worktrees.sh create <name> [base-ref]` instead of bare `git worktree add` followed by a full bootstrap: it seeds the worktree from cached build artifacts (shared read-only store + copies of the yarn layer) in minutes. Upstream artifacts in such worktrees are read-only symlinks — run `scripts/worktrees.sh thaw <path>` before rebuilding an upstream component locally. See `scripts/worktrees.sh --help`.
+</worktrees>
+
 <lockfile_discipline>
 Never bulk-update lockfiles (`Cargo.lock`, `yarn.lock`). Use targeted updates only: `cargo update --precise <version> --package <name>` for Rust, and `yarn up <package>@<version>` in the relevant workspace for TypeScript. Bulk updates drag in unrelated transitive changes that make review impossible and frequently break reproducibility.
 </lockfile_discipline>

barretenberg/cpp/bootstrap.sh

@@ -15,6 +15,12 @@ export native_build_dir=$(scripts/preset-build-dir $native_preset)
 # Uses a sentinel prefix to reliably find the version location, enabling re-injection on cached binaries.
 function inject_version {
   local binary=$1
+  # Read-only binaries are frozen cached-store artifacts (CACHE_LINK_DIR worktrees); stamping them
+  # would mutate state shared across checkouts, so leave them with their as-built version.
+  if [ ! -w "$binary" ]; then
+    echo "Skipping version injection into read-only cached binary $binary." >&2
+    return 0
+  fi
   if semver check "$REF_NAME"; then
     local version=${REF_NAME#v}
   else

barretenberg/ts/bootstrap.sh

@@ -28,14 +28,20 @@ function build {
 
   # We copy snapshot dirs to dest so we can run tests from dest.
   # This is because web-workers run into issues with transpilation.
-  for snapshot_dir in src/**/__snapshots__; do
-    dest_dir="${snapshot_dir/src\//dest\/node\/}"
-    rm -rf "$dest_dir"
-    cp -r "$snapshot_dir" "$dest_dir"
-    for file in $dest_dir/*.test.ts.snap; do
-      mv "$file" "${file/.test.ts.snap/.test.js.snap}"
+  # Skipped when dest is a read-only cached-store symlink (CACHE_LINK_DIR worktrees): bb.js's own
+  # tests can't run from such a checkout anyway, and consumers don't need the snapshots.
+  if [ -w dest/node ]; then
+    for snapshot_dir in src/**/__snapshots__; do
+      dest_dir="${snapshot_dir/src\//dest\/node\/}"
+      rm -rf "$dest_dir"
+      cp -r "$snapshot_dir" "$dest_dir"
+      for file in $dest_dir/*.test.ts.snap; do
+        mv "$file" "${file/.test.ts.snap/.test.js.snap}"
+      done
     done
-  done
+  else
+    echo "Skipping snapshot copy into read-only cached dest." >&2
+  fi
 }
 
 function test_cmds {

ci3/cache_download

@@ -20,11 +20,202 @@ fi
 tar_file="$1"
 out_dir="${2:-.}"
 
+# Link mode: when CACHE_LINK_DIR points at an extracted, read-only, content-addressed store,
+# extract each tarball into the store once and graft symlinks into out_dir instead of extracting
+# in place. Never on CI (the store is a local-dev convenience; CI always extracts in place).
+# Excluded tarballs always extract in place:
+# - yarn-project: contents interleave with tracked src/ files and the generated outputs must stay
+#   writable for incremental rebuilds.
+# - bb.js / noir-packages: their contents are loaded as Node.js modules. Node resolves imports from
+#   a module's REAL path, so store-resident JS cannot see the checkout's node_modules and runtime
+#   dependencies (msgpackr, pako, ...) fail to resolve.
+link_mode=0
+case "$tar_file" in
+  yarn-project*|bb.js-*|noir-packages-*) ;;
+  *)
+    if [[ -n "${CACHE_LINK_DIR:-}" && "${CI:-0}" -ne 1 ]]; then
+      link_mode=1
+    fi
+    ;;
+esac
+
+# Strip the compression suffix to derive the store entry name.
+entry_name="$tar_file"
+entry_name="${entry_name%.tar.gz}"
+entry_name="${entry_name%.zst}"
+entry_name="${entry_name%.tar}"
+
 function extract_tar {
   if [[ "$tar_file" == *.zst ]]; then
-    tar --use-compress-program="zstd -d" -x -C "$out_dir" &>/dev/null
+    tar --use-compress-program="zstd -d" -x -C "$1" &>/dev/null
+  else
+    tar -xzf - -C "$1" &>/dev/null
+  fi
+}
+
+# List the paths inside the tarball (one per line, dirs have a trailing slash, no leading ./).
+function list_tar {
+  if [[ "$tar_file" == *.zst ]]; then
+    tar --use-compress-program="zstd -d" -tf "$1" 2>/dev/null
+  else
+    tar -tzf "$1" 2>/dev/null
+  fi
+}
+
+# Ensure the store entry exists and is frozen (read-only). Concurrency-safe via an mkdir lock so
+# parallel per-contract downloads of the same entry extract it exactly once.
+function ensure_store_entry {
+  local tarfile="$1"
+  local entry_dir="$CACHE_LINK_DIR/$entry_name"
+  [[ -d "$entry_dir" ]] && return 0
+
+  mkdir -p "$CACHE_LINK_DIR"
+  local lock="$CACHE_LINK_DIR/.lock.$entry_name"
+  local waited=0
+  while ! mkdir "$lock" 2>/dev/null; do
+    # Another process is extracting this entry. Wait for it to finish.
+    [[ -d "$entry_dir" ]] && return 0
+    sleep 0.2
+    waited=$((waited + 1))
+    if [[ $waited -gt 600 ]]; then
+      echo_stderr "Timed out waiting for store extraction lock on $entry_name."
+      return 1
+    fi
+  done
+  # shellcheck disable=SC2064
+  trap "rmdir '$lock' 2>/dev/null || true" RETURN
+
+  # Recheck under the lock in case another process won the race before we acquired it.
+  if [[ -d "$entry_dir" ]]; then
+    return 0
+  fi
+
+  local tmp_dir="$CACHE_LINK_DIR/.tmp.$entry_name.$$"
+  rm -rf "$tmp_dir"
+  mkdir -p "$tmp_dir"
+  if ! extract_tar "$tmp_dir" < "$tarfile"; then
+    rm -rf "$tmp_dir"
+    echo_stderr "Failed to extract $tar_file into the store."
+    return 1
+  fi
+  if ! mv "$tmp_dir" "$entry_dir" 2>/dev/null; then
+    # Lost the race: another process created the entry between our recheck and the mv.
+    rm -rf "$tmp_dir"
+    [[ -d "$entry_dir" ]] && return 0
+    echo_stderr "Failed to move store entry into place for $entry_name."
+    return 1
+  fi
+  chmod -R a-w "$entry_dir" 2>/dev/null || true
+}
+
+# Graft symlinks from out_dir into the store entry. For every tarball path we descend through
+# directory components that already exist as REAL descendable directories in out_dir (tracked dirs,
+# uninitialised submodule dirs, thawed local copies) and create an absolute symlink at the first
+# component that is not such a directory (the "link root"). Existing symlinks are repointed; an
+# existing real file/dir at the link root is left alone with a warning (a deliberate local override).
+#
+# A directory in out_dir is "descendable" iff it exists, is a real (non-symlink) dir, AND every one
+# of its ancestors is descendable. The ancestor condition stops us from descending past an existing
+# store symlink: if noir-repo/target is a symlink, paths beneath it resolve through it to real store
+# dirs, but noir-repo/target is not descendable so noir-repo/target stays the single link root.
+#
+# A link root must also be IGNORED by git once created: gitignore patterns with a trailing slash
+# (e.g. barretenberg/cpp's "build*/") match directories only, so a symlink at that path would show
+# up as untracked, dirtying git status and disabling content-hash caching for the whole checkout.
+# When the would-be symlink is not ignored and the store side is a directory, we degrade: create a
+# real directory there and push the link root one level deeper, repeating until the path is ignored
+# (a real dir DOES match dir-only patterns, and everything beneath an ignored dir is ignored).
+#
+# Performance: yarn-project-sized tarballs list tens of thousands of files, but most share a handful
+# of link roots. We walk each path top-down, memoising the descendable decision per directory prefix
+# and recording each link root once, so the work is roughly proportional to the number of distinct
+# directories, not files. The walk derives link roots from the paths themselves and so is correct
+# even when the tarball omits explicit entries for intermediate directories (e.g. a listing that
+# starts at build/bin/ with no bare build/ entry).
+function graft_from_store {
+  local entry_dir="$CACHE_LINK_DIR/$entry_name"
+  local listing
+  listing=$(list_tar "$1")
+  [[ -z "$listing" ]] && return 0
+
+  # descendable[prefix]: 1 = real descendable dir, 0 = not (memoised, including negative results).
+  local -A descendable=()
+  # seen_root[prefix]: a link root we have already created a symlink for.
+  local -A seen_root=()
+  local p
+  while IFS= read -r p; do
+    p="${p%/}"
+    [[ -z "$p" ]] && continue
+
+    # Re-walk the same path after each ignore-degradation; bounded by the path's depth.
+    while :; do
+      # Walk components top-down until we hit the first non-descendable prefix (the link root).
+      local acc="" comp link_root="" decided=0
+      IFS='/' read -ra parts <<<"$p"
+      for comp in "${parts[@]}"; do
+        acc="${acc:+$acc/}$comp"
+        local d="${descendable[$acc]:-}"
+        if [[ -z "$d" ]]; then
+          if [[ -d "$out_dir/$acc" && ! -L "$out_dir/$acc" ]]; then
+            d=1
+          else
+            d=0
+          fi
+          descendable["$acc"]=$d
+        fi
+        if [[ "$d" -eq 0 ]]; then
+          link_root="$acc"
+          decided=1
+          break
+        fi
+      done
+      # Whole path is descendable (all components are real dirs already) -> nothing to link.
+      [[ "$decided" -eq 0 ]] && break
+      [[ -n "${seen_root[$link_root]:-}" ]] && break
+
+      local dest="$out_dir/$link_root"
+      local target="$entry_dir/$link_root"
+      if [[ -e "$dest" && ! -L "$dest" ]]; then
+        echo_stderr "Not grafting $link_root: a real file/dir already exists in $out_dir (local override)."
+        seen_root["$link_root"]=1
+        break
+      fi
+
+      # check-ignore exits 0 = ignored, 1 = not ignored, 128 = error (e.g. not in a git repo).
+      # Run from the link root's parent so paths inside an initialised submodule (noir-repo) are
+      # checked against the submodule's own ignore rules, not the parent repo's. A nonexistent path
+      # is evaluated as a file, so dir-only patterns report "not ignored" and we degrade below.
+      local ignore_rc=0
+      git -C "$out_dir/$(dirname "$link_root")" check-ignore -q "$(basename "$link_root")" 2>/dev/null || ignore_rc=$?
+      if [[ "$ignore_rc" -eq 1 && -d "$target" ]]; then
+        # Degrade: real dir at the link root (matches dir-only ignore patterns), link one level deeper.
+        [[ -L "$dest" ]] && rm -f "$dest"
+        mkdir -p "$dest"
+        descendable["$link_root"]=1
+        continue
+      fi
+      if [[ "$ignore_rc" -eq 1 ]]; then
+        echo_stderr "WARNING: grafted $link_root is not gitignored; it will show as untracked."
+      fi
+      mkdir -p "$(dirname "$dest")"
+      ln -sfn "$target" "$dest"
+      seen_root["$link_root"]=1
+      break
+    done
+  done <<<"$listing"
+
+  # Record the linked entry for gc, crash-safe append (one entry name per line).
+  echo "$entry_name" >> "$root/.deps-manifest.linked"
+}
+
+# Place the obtained tarball file: graft symlinks in link mode, otherwise extract in place.
+function place_tar {
+  local tarfile="$1"
+  if [[ "$link_mode" -eq 1 ]]; then
+    ensure_store_entry "$tarfile" || return 1
+    graft_from_store "$tarfile" || return 1
   else
-    tar -xzf - -C "$out_dir" &>/dev/null
+    extract_tar "$out_dir" < "$tarfile" || return 1
   fi
 }
 
@@ -65,7 +256,7 @@ if [[ -n "${CACHE_LOCAL_DIR:-}" ]]; then
 
   if [[ -f "$local_cache_file" ]]; then
     echo_stderr "Local cache hit for $tar_file."
-    extract_tar < "$local_cache_file"
+    place_tar "$local_cache_file"
     echo_stderr "Cache extraction of $tar_file from local cache complete in ${SECONDS}s."
     exit 0
   fi
@@ -77,12 +268,29 @@ if [[ -n "${CACHE_LOCAL_DIR:-}" ]]; then
     exit 1
   fi
 
-  extract_tar < "$local_cache_file"
+  place_tar "$local_cache_file"
+  echo_stderr "Cache download and extraction of $tar_file complete in ${SECONDS}s."
+  exit 0
+fi
+
+# No local tarball cache. Link mode needs the tarball as a file (to extract into the store and to
+# list its paths), so stream into a temp file; otherwise extract straight from the download stream.
+if [[ "$link_mode" -eq 1 ]]; then
+  tmp_tar=$(mktemp)
+  trap 'rm -f "$tmp_tar"' EXIT
+  if ! download_from_remote > "$tmp_tar"; then
+    echo_stderr "Cache download of $tar_file failed."
+    exit 1
+  fi
+  if ! place_tar "$tmp_tar"; then
+    echo_stderr "Cache download of $tar_file failed."
+    exit 1
+  fi
   echo_stderr "Cache download and extraction of $tar_file complete in ${SECONDS}s."
   exit 0
 fi
 
-if ! download_from_remote | extract_tar; then
+if ! download_from_remote | extract_tar "$out_dir"; then
   echo_stderr "Cache download of $tar_file failed."
   exit 1
 fi

ci3/cache_local.test.sh

@@ -204,6 +204,36 @@ test_inaccessible_cache_dir_falls_through() {
   fi
 }
 
+test_upload_local_save_without_ci() {
+  log "\nTest 9: cache_upload saves to local cache with CI=0 and no S3_FORCE_UPLOAD"
+
+  export CACHE_LOCAL_DIR="$test_root/local-cache-ci0"
+  mkdir -p "$CACHE_LOCAL_DIR"
+
+  local stderr_output
+  stderr_output=$(CI=0 "$script_dir/cache_upload" "test-ci0.tar.gz" "$test_root/source/file1.txt" 2>&1 >/dev/null) || true
+
+  if [[ -f "$CACHE_LOCAL_DIR/test-ci0.tar.gz" ]]; then
+    pass "Local build populated local cache without CI/S3_FORCE_UPLOAD"
+  else
+    fail "Local build did not populate local cache (got: $stderr_output)"
+  fi
+  if echo "$stderr_output" | grep -q "Skipping S3 upload"; then
+    pass "S3 upload still skipped at CI=0"
+  else
+    fail "Expected S3 upload skip at CI=0 (got: $stderr_output)"
+  fi
+
+  # With no CACHE_LOCAL_DIR either, upload is a no-op and skips tarring entirely.
+  unset CACHE_LOCAL_DIR
+  stderr_output=$(CI=0 "$script_dir/cache_upload" "test-ci0-noop.tar.gz" "$test_root/source/file1.txt" 2>&1 >/dev/null) || true
+  if echo "$stderr_output" | grep -q "no CACHE_LOCAL_DIR"; then
+    pass "No-op exit when there is nowhere to save"
+  else
+    fail "Expected no-op exit message (got: $stderr_output)"
+  fi
+}
+
 main() {
   log "=== Local Cache Test Suite ===\n"
 
@@ -217,6 +247,7 @@ main() {
   test_roundtrip
   test_disabled_cache_skips_local
   test_inaccessible_cache_dir_falls_through
+  test_upload_local_save_without_ci
 
   log "\n=== Results ==="
   echo -e "\033[32mPassed: $passed\033[0m"