diff --git a/Containerfile b/Containerfile index a8ab1bb76..76e7b8dfa 100644 --- a/Containerfile +++ b/Containerfile @@ -35,7 +35,149 @@ RUN cc -Wall -Werror -g /usr/local/src/su-exec/su-exec.c -o /usr/local/bin/su-ex ## Chef Prepare (look at project and see wat we need) FROM chef AS recipe WORKDIR /build/src -COPY . /build/src +# Manifest-only copy: `cargo chef prepare` only needs Cargo.toml manifests and Cargo.lock +# to build recipe.json — it does not read any .rs source files. +# Copying the full source tree here would cause Docker to invalidate this layer (and +# therefore the expensive `cargo chef cook` dependency layers) on every source-code change. +# By copying only manifests, the cook layers stay cached for source-only edits. +# +# MAINTENANCE: Keep this list in sync with all in-repo path crates (packages/, console/, +# contrib/). This includes the root crate itself plus every crate reachable as a path +# dependency from the root — i.e. all packages discovered by `cargo metadata --no-deps` +# whose manifest path is inside this repository. Note: the `[workspace].members` key in +# the root Cargo.toml only lists packages not auto-discovered via path dependencies; it +# is a much smaller set and should NOT be used as the authoritative list here. +# Every new in-repo path crate must have a corresponding COPY line added; every removed +# or moved crate must have its line updated or removed accordingly. +COPY Cargo.toml Cargo.lock ./ +COPY console/tracker-client/Cargo.toml console/tracker-client/ +COPY contrib/bencode/Cargo.toml contrib/bencode/ +COPY contrib/dev-tools/analysis/workspace-coupling/Cargo.toml contrib/dev-tools/analysis/workspace-coupling/ +COPY packages/axum-health-check-api-server/Cargo.toml packages/axum-health-check-api-server/ +COPY packages/axum-http-server/Cargo.toml packages/axum-http-server/ +COPY packages/axum-rest-api-server/Cargo.toml packages/axum-rest-api-server/ +COPY packages/axum-server/Cargo.toml packages/axum-server/ +COPY packages/clock/Cargo.toml packages/clock/ +COPY packages/configuration/Cargo.toml packages/configuration/ +COPY packages/events/Cargo.toml packages/events/ +COPY packages/http-protocol/Cargo.toml packages/http-protocol/ +COPY packages/http-tracker-core/Cargo.toml packages/http-tracker-core/ +COPY packages/located-error/Cargo.toml packages/located-error/ +COPY packages/metrics/Cargo.toml packages/metrics/ +COPY packages/net-primitives/Cargo.toml packages/net-primitives/ +COPY packages/peer-id/Cargo.toml packages/peer-id/ +COPY packages/primitives/Cargo.toml packages/primitives/ +COPY packages/rest-api-client/Cargo.toml packages/rest-api-client/ +COPY packages/rest-api-core/Cargo.toml packages/rest-api-core/ +COPY packages/server-lib/Cargo.toml packages/server-lib/ +COPY packages/swarm-coordination-registry/Cargo.toml packages/swarm-coordination-registry/ +COPY packages/test-helpers/Cargo.toml packages/test-helpers/ +COPY packages/torrent-repository-benchmarking/Cargo.toml packages/torrent-repository-benchmarking/ +COPY packages/tracker-client/Cargo.toml packages/tracker-client/ +COPY packages/tracker-core/Cargo.toml packages/tracker-core/ +COPY packages/udp-protocol/Cargo.toml packages/udp-protocol/ +COPY packages/udp-server/Cargo.toml packages/udp-server/ +COPY packages/udp-tracker-core/Cargo.toml packages/udp-tracker-core/ +# Create stub source files for every in-repo target. +# `cargo chef prepare` runs `cargo metadata` internally, which requires every +# package to have at least one resolvable target file on disk — whether the +# target is explicitly declared in Cargo.toml (e.g. [lib], [[bin]], [[bench]]) +# or auto-detected by Cargo (e.g. src/lib.rs, src/main.rs, src/bin/*.rs). +# Packages with no source files at all cause `cargo metadata` to abort with +# "no targets specified in the manifest". Examples and tests also need stubs +# when auto-detected, because Cargo validates them during manifest loading. +# +# The canonical list below was derived from: +# cargo metadata --no-deps --format-version 1 | jq -r '.packages[].targets[].src_path' +# filtered to paths inside this repository. Re-run that command whenever a +# new package, binary, example, or bench target is added to the workspace and +# add the corresponding mkdir / touch lines here. +# +# MAINTENANCE: When adding a new in-repo crate or target, add the corresponding +# stub lines below AND the Cargo.toml COPY line in the manifest-only block above. +RUN mkdir -p \ + src/bin \ + contrib/bencode/src \ + contrib/bencode/benches \ + contrib/dev-tools/analysis/workspace-coupling/src \ + console/tracker-client/src/bin \ + packages/axum-health-check-api-server/src \ + packages/axum-http-server/src \ + packages/axum-http-server/examples \ + packages/axum-rest-api-server/src \ + packages/axum-server/src \ + packages/clock/src \ + packages/configuration/src \ + packages/events/src \ + packages/http-protocol/src \ + packages/http-tracker-core/src \ + packages/http-tracker-core/benches \ + packages/located-error/src \ + packages/metrics/src \ + packages/net-primitives/src \ + packages/peer-id/src \ + packages/primitives/src \ + packages/rest-api-client/src \ + packages/rest-api-core/src \ + packages/server-lib/src \ + packages/swarm-coordination-registry/src \ + packages/test-helpers/src \ + packages/torrent-repository-benchmarking/src \ + packages/torrent-repository-benchmarking/benches \ + packages/tracker-client/src \ + packages/tracker-core/src \ + packages/tracker-core/src/bin \ + packages/udp-protocol/src \ + packages/udp-server/src \ + packages/udp-server/examples \ + packages/udp-tracker-core/src \ + packages/udp-tracker-core/benches \ + && touch \ + src/lib.rs \ + src/main.rs \ + src/bin/e2e_tests_runner.rs \ + src/bin/http_health_check.rs \ + src/bin/profiling.rs \ + src/bin/qbittorrent_e2e_runner.rs \ + contrib/bencode/src/lib.rs \ + contrib/bencode/benches/bencode_benchmark.rs \ + contrib/dev-tools/analysis/workspace-coupling/src/main.rs \ + console/tracker-client/src/lib.rs \ + console/tracker-client/src/bin/http_tracker_client.rs \ + console/tracker-client/src/bin/tracker_checker.rs \ + console/tracker-client/src/bin/tracker_client.rs \ + console/tracker-client/src/bin/udp_tracker_client.rs \ + packages/axum-health-check-api-server/src/lib.rs \ + packages/axum-http-server/src/lib.rs \ + packages/axum-http-server/examples/http_only_public_tracker.rs \ + packages/axum-rest-api-server/src/lib.rs \ + packages/axum-server/src/lib.rs \ + packages/clock/src/lib.rs \ + packages/configuration/src/lib.rs \ + packages/events/src/lib.rs \ + packages/http-protocol/src/lib.rs \ + packages/http-tracker-core/src/lib.rs \ + packages/http-tracker-core/benches/http_tracker_core_benchmark.rs \ + packages/located-error/src/lib.rs \ + packages/metrics/src/lib.rs \ + packages/net-primitives/src/lib.rs \ + packages/peer-id/src/lib.rs \ + packages/primitives/src/lib.rs \ + packages/rest-api-client/src/lib.rs \ + packages/rest-api-core/src/lib.rs \ + packages/server-lib/src/lib.rs \ + packages/swarm-coordination-registry/src/lib.rs \ + packages/test-helpers/src/lib.rs \ + packages/torrent-repository-benchmarking/src/lib.rs \ + packages/torrent-repository-benchmarking/benches/repository_benchmark.rs \ + packages/tracker-client/src/lib.rs \ + packages/tracker-core/src/lib.rs \ + packages/tracker-core/src/bin/persistence_benchmark_runner.rs \ + packages/udp-protocol/src/lib.rs \ + packages/udp-server/src/lib.rs \ + packages/udp-server/examples/udp_only_public_tracker.rs \ + packages/udp-tracker-core/src/lib.rs \ + packages/udp-tracker-core/benches/udp_tracker_core_benchmark.rs RUN cargo chef prepare --recipe-path /build/recipe.json diff --git a/docs/issues/open/1852-1840-workflow-performance-recipe-stage-manifest-only-copy/ISSUE.md b/docs/issues/open/1852-1840-workflow-performance-recipe-stage-manifest-only-copy/ISSUE.md index bf8ff7f1c..2605e60c1 100644 --- a/docs/issues/open/1852-1840-workflow-performance-recipe-stage-manifest-only-copy/ISSUE.md +++ b/docs/issues/open/1852-1840-workflow-performance-recipe-stage-manifest-only-copy/ISSUE.md @@ -166,14 +166,15 @@ A CI check that validates all workspace member directories have a corresponding Status values: `TODO`, `IN_PROGRESS`, `BLOCKED`, `DONE`. -| ID | Status | Task | Notes / Expected Output | -| --- | ------ | -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| T1 | TODO | Replace full-tree COPY with manifest-only COPY in recipe stage | One `COPY /` line per workspace member, plus root `Cargo.toml` and `Cargo.lock`. | -| T2 | TODO | Verify recipe.json equivalence | Build locally; diff `recipe.json` output before and after to confirm it is identical. | -| T3 | TODO | Verify full build pipeline | Run `docker build --target release .` locally; confirm all stages succeed. | -| T4 | TODO | Measure warm build time improvement | Run warm baseline (`run-container-baseline.sh`) with a source-only change; confirm cook layers show cache hit; record time saved. | -| T5 | TODO | Document maintenance requirement in Containerfile | Add inline comment above the manifest COPY block explaining the sync requirement. | -| T6 | TODO | Optionally add CI drift check | Script or CI step that compares workspace members in `Cargo.toml` against `COPY` lines in `Containerfile` and fails on mismatch. | +| ID | Status | Task | Notes / Expected Output | +| --- | ------ | ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| T1 | DONE | Replace full-tree COPY with manifest-only COPY in recipe stage | One `COPY /` line per in-repo path crate (packages/, console/, contrib/), plus root `Cargo.toml` and `Cargo.lock`. | +| T2 | TODO | Verify recipe.json equivalence | Build locally; diff `recipe.json` output before and after to confirm it is identical. | +| T3 | TODO | Verify full build pipeline | Run `docker build --target release .` locally; confirm all stages succeed. | +| T4 | TODO | Measure warm build time improvement | Run warm baseline (`run-container-baseline.sh`) with a source-only change; confirm cook layers show cache hit; record time saved. | +| T5 | DONE | Document maintenance requirement in Containerfile | Add inline comment above the manifest COPY block explaining the sync requirement. | +| T6 | TODO | Optionally add CI drift check | Script or CI step that compares workspace members in `Cargo.toml` against `COPY` lines in `Containerfile` and fails on mismatch. | +| T7 | DONE | Add source file stubs to recipe stage to fix `cargo metadata` validation | `cargo chef prepare` calls `cargo metadata` internally, which validates that all declared targets have source files present. Add a `RUN mkdir -p / touch` step for every declared target path. | ## Progress Tracking @@ -196,14 +197,16 @@ Status values: `TODO`, `IN_PROGRESS`, `BLOCKED`, `DONE`. Append one line per meaningful update. - 2026-06-01 00:00 UTC - GitHub Copilot - Drafted recipe stage manifest-only copy issue from EPIC #1840 discussion - draft file created +- 2026-06-01 00:00 UTC - GitHub Copilot - Implemented T1 and T5: replaced full-tree COPY with manifest-only COPY in Containerfile recipe stage; added maintenance comment +- 2026-06-01 00:00 UTC - GitHub Copilot - Implemented T7: added source file stubs RUN step; cargo metadata validation fix for recipe stage ## Acceptance Criteria -- [ ] AC1: The `recipe` stage uses manifest-only COPY (no full-tree copy); every workspace member `Cargo.toml` and root `Cargo.lock` is explicitly listed. +- [x] AC1: The `recipe` stage uses manifest-only COPY (no full-tree copy); every workspace member `Cargo.toml` and root `Cargo.lock` is explicitly listed. - [ ] AC2: `recipe.json` produced by the new stage is identical to the one produced by the old full-tree copy stage (verified by diff). - [ ] AC3: Full build pipeline (`docker build --target release .`) completes successfully with no regressions. - [ ] AC4: Warm baseline run with a source-only change shows cook layers hitting cache; time saved is recorded. -- [ ] AC5: Containerfile contains an inline comment documenting the manifest list maintenance requirement. +- [x] AC5: Containerfile contains an inline comment documenting the manifest list maintenance requirement. - [ ] `linter all` exits with code `0` - [ ] All CI checks pass for the changed `Containerfile` - [ ] Manual verification scenarios are executed and documented (status + evidence) @@ -231,10 +234,10 @@ Status values: `TODO`, `IN_PROGRESS`, `DONE`, `FAILED`, `BLOCKED`. ### Acceptance Verification -| AC ID | Status (`TODO`/`DONE`) | Evidence | -| ----- | ---------------------- | ---------------- | -| AC1 | TODO | {diff link} | -| AC2 | TODO | {diff link} | -| AC3 | TODO | {CI run link} | -| AC4 | TODO | {benchmark link} | -| AC5 | TODO | {diff link} | +| AC ID | Status (`TODO`/`DONE`) | Evidence | +| ----- | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| AC1 | DONE | Containerfile recipe stage updated; all 28 in-repo path-crate Cargo.toml files (packages/, console/, contrib/) plus root Cargo.toml and Cargo.lock are listed (verified via `cargo metadata --no-deps`) | +| AC2 | TODO | {diff link} | +| AC3 | TODO | {CI run link} | +| AC4 | TODO | {benchmark link} | +| AC5 | DONE | Maintenance comment block added above the COPY lines in Containerfile |