From 50708d7637d628d30fc31921f6ae7ebf2787b5a1 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 15 Mar 2026 13:29:22 -0700 Subject: [PATCH 1/6] docs: rewrite roadmap around dual CLI surfaces --- README.md | 2 +- ROADMAP.md | 1896 ++--------------- STATUS.md | 124 +- .../services/rotateVaultPassphrase.test.js | 20 +- .../ContentAddressableStore.rotation.test.js | 4 +- test/unit/vault/VaultService.test.js | 4 +- 6 files changed, 260 insertions(+), 1790 deletions(-) diff --git a/README.md b/README.md index 7f39a3f..5f2ea60 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ We use the object database. - **Lifecycle management** `readManifest`, `inspectAsset`, `collectReferencedChunks` — inspect trees, plan deletions, audit storage. - **Vault** GC-safe ref-based storage. One ref (`refs/cas/vault`) indexes all assets by slug. No more silent data loss from `git gc`. - **Interactive dashboard** `git cas inspect` with chunk heatmap, animated progress bars, and rich manifest views. -- **Verify & JSON output** `git cas verify` checks integrity; `--json` on all commands for CI/scripting. +- **Verify & JSON output** `git cas verify` checks integrity; `--json` on all current human-facing commands provides convenience structured output for CI/scripting. **Use it for:** binary assets, build artifacts, model weights, data packs, secret bundles, weird experiments, etc. diff --git a/ROADMAP.md b/ROADMAP.md index 81fdffc..aca7122 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,1780 +1,252 @@ # @git-stunts/cas — ROADMAP -Content-addressed storage backed by Git's object database (ODB), with optional encryption and pluggable codecs. - -This roadmap is structured as: - -1. **Header** — Platform, dependencies, supported environments -2. **Canonical CasError Codes** — Single registry of all error codes referenced by tasks -3. **Contracts** — Return/throw semantics for all public methods -4. **Version Plan** — Table mapping versions to milestones -5. **Milestone Dependency Graph** — ASCII diagram -6. **Milestones & Task Cards** — 8 milestones (7 closed, 1 open), remaining task cards -7. **Feature Matrix** — Competitive landscape vs. Git LFS, git-annex, Restic, Age, DVC -8. **Competitive Analysis** — When to use git-cas and when not to, with concrete scenarios - ---- - -## 1) Platform & Supported Environments - -### Supported runtimes -- Node.js: **22.x** (primary target) -- OS: Linux (CI), macOS (dev), Windows (best-effort; Git plumbing assumptions apply) - -### External dependencies / assumptions -- Requires `git` available on PATH for adapter-backed operations (integration tests and real persistence). -- Uses Git plumbing commands via `cat-file`, `hash-object`, `ls-tree`, etc. -- Encryption uses **AES-256-GCM** (requires 32-byte key). -- Manifests validated by Zod schemas; malformed manifests must fail closed. - -### Design constraints (non-negotiable) -- Git objects are immutable; "rollback" is conceptual (unreachable objects are GC'd). -- Integrity is enforced via SHA-256 digests per chunk and GCM auth tag for encrypted restores. -- APIs are additive in v1.x; any manifest-format break is reserved for v2.0.0. - ---- - -## 2) Canonical CasError Codes - -Single registry of all error codes used across the codebase. Each code is a string passed as the `code` argument to `new CasError(message, code, meta)`. - -| Code | Description | Planned By | -|------|-------------|------------| -| `INVALID_KEY_LENGTH` | Encryption key is not exactly 32 bytes (AES-256 requirement). Error meta includes `{ expected: 32, actual: }`. | v1.1.0 | -| `INVALID_KEY_TYPE` | Encryption key is not a Buffer or Uint8Array. | v1.1.0 | -| `INTEGRITY_ERROR` | Decryption auth-tag verification failed (wrong key, tampered ciphertext, or tampered tag), or chunk digest mismatch on restore. | v1.1.0 | -| `STREAM_ERROR` | Read stream failed during `storeFile`. Partial chunks may have been written to Git ODB (unreachable; handled by `git gc`). Meta includes `{ chunksWritten: }`. | v1.2.0 | -| `MISSING_KEY` | Encryption key required to restore encrypted content but none was provided. | v1.2.0 | -| `TREE_PARSE_ERROR` | `git ls-tree` output could not be parsed into valid entries. | v1.2.0 | -| `MANIFEST_NOT_FOUND` | No manifest entry (e.g. `manifest.json` / `manifest.cbor`) found in the Git tree. | v1.4.0 | -| `GIT_ERROR` | Underlying Git plumbing command failed. Wraps the original error from the plumbing layer. | v1.2.0 | -| `INVALID_CHUNKING_STRATEGY` | Manifest contains unrecognized chunking strategy (not `fixed` or `cdc`). | Task 10.3 | -| `NO_MATCHING_RECIPIENT` | No recipient entry matches the provided KEK. Caller's key is not in the recipient list. | Task 11.1 | -| `DEK_UNWRAP_FAILED` | Failed to unwrap DEK with the provided KEK. Wrong key or tampered wrappedDek. | Task 11.1 | -| `RECIPIENT_NOT_FOUND` | Recipient label not found in manifest recipient list. | Task 11.2 | -| `RECIPIENT_ALREADY_EXISTS` | Recipient label already exists in manifest. | Task 11.2 | -| `CANNOT_REMOVE_LAST_RECIPIENT` | Cannot remove the last recipient — at least one must remain. | Task 11.2 | -| `ROTATION_NOT_SUPPORTED` | Key rotation requires envelope encryption (DEK/KEK model). Legacy manifests must be re-stored. | Task 12.1 | -| `STREAM_NOT_CONSUMED` | `finalize()` called on encryption stream before the generator was fully consumed. | v4.0.1 | -| `RESTORE_TOO_LARGE` | Encrypted/compressed file exceeds `maxRestoreBufferSize`. Buffered restore would OOM. Suggest increasing limit or storing without encryption. | M16 | -| `ENCRYPTION_BUFFER_EXCEEDED` | Web Crypto adapter accumulated buffer exceeds limit during streaming encryption (Deno-specific). Suggest Node.js/Bun or unencrypted store. | M16 | - ---- - -## 3) Contracts - -Return and throw semantics for every public method (current and planned). - -### `storeFile({ filePath, slug, filename, encryptionKey? })` -- **Returns:** `Promise` — frozen, Zod-validated value object. -- **Throws:** `CasError('INVALID_KEY_LENGTH')` if `encryptionKey` is provided and `length !== 32`. -- **Throws:** `CasError('INVALID_KEY_TYPE')` if `encryptionKey` is not a Buffer. -- **Throws:** `CasError('STREAM_ERROR')` if the read stream fails mid-store. No manifest is returned; partial blobs may remain in Git ODB. -- **Throws:** Node.js filesystem error if `filePath` does not exist or is unreadable. -- **Empty file:** Returns `Manifest { size: 0, chunks: [] }` with no blob writes for chunk content. - -### `restoreFile({ manifest, encryptionKey?, outputPath })` -- **Returns:** `Promise<{ bytesWritten: number }>`. -- **Throws:** `CasError('INTEGRITY_ERROR')` if any chunk's SHA-256 digest does not match `chunk.digest`. -- **Throws:** `CasError('INTEGRITY_ERROR')` if decryption fails (wrong key or tampered ciphertext). -- **Throws:** `CasError('INVALID_KEY_LENGTH')` if `encryptionKey` is provided and `length !== 32`. -- **Empty manifest:** Creates a 0-byte file at `outputPath`. - -### `encrypt({ buffer, key })` -- **Returns:** `{ buf: Buffer, meta: { algorithm: 'aes-256-gcm', nonce: string, tag: string, encrypted: true } }`. -- **Throws:** `CasError('INVALID_KEY_LENGTH')` if `key.length !== 32`. -- **Throws:** `CasError('INVALID_KEY_TYPE')` if `key` is not a Buffer. - -### `decrypt({ buffer, key, meta })` -- **Returns:** `Buffer` — original plaintext. -- **Passthrough:** If `meta.encrypted` is falsy or `meta` is undefined, returns `buffer` unchanged. -- **Throws:** `CasError('INTEGRITY_ERROR')` if GCM auth-tag verification fails. - -### `createTree({ manifest })` -- **Returns:** `Promise` — Git OID of the created tree. -- **Throws:** Zod validation error if `manifest` is invalid. - -### `readManifest({ treeOid })` -- **Returns:** `Promise` — frozen, Zod-validated value object. -- **Throws:** `CasError('MANIFEST_NOT_FOUND')` if no manifest entry exists in the tree. -- **Throws:** `CasError('GIT_ERROR')` if the underlying Git command fails. -- **Throws:** Zod validation error if the manifest blob is corrupt. - -### `verifyIntegrity(manifest)` -- **Returns:** `Promise` — `true` if all chunk digests match, `false` otherwise. -- **Does not throw** on mismatch; returns `false`. - -### `deleteAsset({ treeOid })` -- **Returns:** `Promise<{ chunksOrphaned: number, slug: string }>`. -- **Throws:** `CasError('MANIFEST_NOT_FOUND')` (delegates to `readManifest`). -- **Side effects:** None. Caller must remove refs; physical deletion requires `git gc --prune`. - -### `findOrphanedChunks({ treeOids })` -- **Returns:** `Promise<{ referenced: Set, total: number }>`. -- **Throws:** `CasError('MANIFEST_NOT_FOUND')` if any `treeOid` lacks a manifest (fail closed). -- **Side effects:** None. Analysis only. - -### `deriveKey({ passphrase, salt?, algorithm?, iterations? })` -- **Returns:** `Promise<{ key: Buffer, salt: Buffer, params: object }>`. -- **Algorithms:** `pbkdf2` (default), `scrypt` — both Node.js built-ins. -- **Throws:** Standard Node.js crypto errors on invalid parameters. - -### CLI: `git cas store --slug [--key-file ]` -- **Output:** Prints manifest JSON to stdout. If `--tree` is passed, prints only the Git tree OID instead. -- **Exit 0:** Store succeeded. -- **Exit 1:** Store failed (error message to stderr). - -### CLI: `git cas tree --manifest ` -- **Output:** Prints Git tree OID to stdout. -- **Exit 0:** Tree created. -- **Exit 1:** Invalid manifest or Git error (message to stderr). - -### CLI: `git cas restore --out [--key-file ]` -- **Output:** Writes restored file to `--out` path. -- **Exit 0:** Restore succeeded, prints bytes written to stdout. -- **Exit 1:** Integrity error, missing manifest, or I/O error (message to stderr). - -### `restoreStream({ manifest, encryptionKey?, passphrase? })` *(implemented — v4.0.0)* -- **Returns:** `AsyncIterable` — verified, decrypted, decompressed chunks in index order. -- **Throws:** `CasError('INTEGRITY_ERROR')` if any chunk fails verification (iteration stops). -- **Throws:** `CasError('MISSING_KEY')` if encrypted and no key provided. -- **Memory:** O(chunkSize) — never buffers full file. - -### `rotateKey({ manifest, oldKey, newKey, label? })` *(implemented — v5.2.0)* -- **Returns:** `Promise` — updated manifest with re-wrapped DEK and incremented `keyVersion`. -- **Throws:** `CasError('DEK_UNWRAP_FAILED')` if `oldKey` cannot unwrap the DEK. -- **Throws:** `CasError('ROTATION_NOT_SUPPORTED')` if manifest uses legacy (non-envelope) encryption. -- **Side effects:** None. Caller must persist via `createTree()`. - -### `addRecipient({ manifest, existingKey, newRecipientKey, label })` *(implemented — v5.1.0)* -- **Returns:** `Promise` — updated manifest with additional recipient entry. -- **Throws:** `CasError('DEK_UNWRAP_FAILED')` if `existingKey` is wrong. -- **Throws:** `CasError('RECIPIENT_ALREADY_EXISTS')` if `label` already exists. -- **Side effects:** None. Caller must persist. - -### `removeRecipient({ manifest, label })` *(implemented — v5.1.0)* -- **Returns:** `Promise` — updated manifest without the named recipient. -- **Throws:** `CasError('RECIPIENT_NOT_FOUND')` if `label` not in recipient list. -- **Throws:** `CasError('CANNOT_REMOVE_LAST_RECIPIENT')` if only 1 recipient remains. - -### CLI: `git cas verify --oid | --slug ` *(implemented — v4.0.1)* -- **Output:** `ok` on success, `fail` on failure. -- **Exit 0:** All chunks verified. -- **Exit 1:** Verification failed or error. - -### CLI: `git cas rotate --slug --old-key-file --new-key-file ` *(implemented — v5.2.0)* -- **Output:** New tree OID on success. -- **Exit 0:** Rotation succeeded, vault updated. -- **Exit 1:** Wrong old key, unsupported manifest, or vault error. - -### CLI: `git cas vault dashboard` *(implemented)* -- **Output:** Interactive full-screen TUI in TTY mode; static table in non-TTY. -- **Exit 0:** User quit normally. -- **Exit 1:** Vault ref missing or error. - -### CLI: `git cas inspect --slug | --oid [--heatmap]` *(implemented)* -- **Output:** Structured manifest anatomy view in TTY; JSON dump in non-TTY. -- **Exit 0:** Manifest read and displayed. -- **Exit 1:** Manifest not found or error. - -### CLI: `git cas vault history --pretty` *(implemented)* -- **Output:** Color-coded timeline in TTY; plain `git log --oneline` without `--pretty`. -- **Exit 0:** History displayed. -- **Exit 1:** Vault ref missing or error. - ---- - -## 4) Version Plan +This document tracks the real current state of `git-cas` and the sequenced work that remains. +Completed milestone detail lives in [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). Superseded work +lives in [GRAVEYARD.md](./GRAVEYARD.md). -| Version | Milestone | Codename | Theme | Status | -|--------:|-----------|----------|-------|--------| -| v4.0.1 | M8+M9 | Spit Shine + Cockpit | CryptoPort refactor, verify, --json, error handler, vault list | ✅ | -| v4.0.0 | M14 | Conduit | Streaming I/O, observability, parallel chunks | ✅ | -| v3.1.0 | M13 | Bijou | TUI dashboard & progress | ✅ | -| v5.0.0 | M10 | Hydra | Content-defined chunking | ✅ | -| v5.1.0 | M11 | Locksmith | Multi-recipient encryption | ✅ | -| v5.2.0 | M12 | Carousel | Key rotation | ✅ | -| v5.3.0 | M16 | Capstone | Audit remediation — all CODE-EVAL.md findings | 🔲 | - ---- - -## 5) Milestone Dependency Graph - -```text -M7 Horizon (v2.0.0) ✅ -M13 Bijou (v3.1.0) ✅ -M14 Conduit (v4.0.0) ✅ -M8 Spit Shine + M9 Cockpit (v4.0.1) ✅ - -M10 Hydra ──────────── ✅ v5.0.0 -M11 Locksmith ──────── ✅ v5.1.0 - └──► M12 Carousel ── ✅ v5.2.0 -M15 Prism ─────────────── ✅ - └──► M16 Capstone ────── 🔲 v5.3.0 -``` - ---- - -## 6) Milestones & Task Cards - -### Milestones at a glance - -| # | Codename | Theme | Version | Tasks | ~LoC | ~Hours | Status | -|---:|--------------|----------------------------|:-------:|------:|-------:|------:|:------:| -| M14| Conduit | Streaming I/O, observability, parallel chunks | v4.0.0 | 4 | ~600 | ~18h | ✅ CLOSED | -| M13| Bijou | TUI dashboard & progress | v3.1.0 | 6 | ~650 | ~20h | ✅ CLOSED | -| M8 | Spit Shine | Review fixups | v4.0.1 | 2 | ~150 | ~3h | ✅ CLOSED | -| M9 | Cockpit | CLI improvements | v4.0.1 | 4 | ~190 | ~5h | ✅ CLOSED | -| M10| Hydra | Content-defined chunking | v5.0.0 | 4 | ~690 | ~22h | ✅ CLOSED | -| M11| Locksmith | Multi-recipient encryption | v5.1.0 | 4 | ~580 | ~20h | ✅ CLOSED | -| M12| Carousel | Key rotation | v5.2.0 | 4 | ~400 | ~13h | ✅ CLOSED | -| M16| Capstone | Audit remediation | v5.3.0 | 13 | ~698 | ~21h | 🔲 OPEN | - -Completed task cards are in [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). Superseded tasks are in [GRAVEYARD.md](./GRAVEYARD.md). - ---- - -# M14 — Conduit (v4.0.0) ✅ CLOSED - -All tasks completed (14.1–14.4). See [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). - ---- - -# M8 — Spit Shine (v4.0.1) ✅ CLOSED - -All tasks completed (8.2–8.3). See [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). - ---- - -# M9 — Cockpit (v4.0.1) ✅ CLOSED - -All tasks completed (9.2–9.5). See [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). - ---- - -# M10 — Hydra (v5.0.0) ✅ CLOSED - -All tasks completed (10.1–10.4). See [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). - -# M11 — Locksmith (v5.1.0) ✅ CLOSED - -All tasks completed (11.1–11.4). See [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). - ---- - -# M12 — Carousel (v5.2.0) ✅ CLOSED - -All tasks completed (12.1–12.4). See [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). - ---- - -# M16 — Capstone (v5.3.0) 🔲 OPEN - -Remediation milestone addressing all negative findings from the [CODE-EVAL.md](./CODE-EVAL.md) forensic architectural audit. Covers 9 code flaws (Phase 2), 7 pre-existing concerns (C1–C7), and 3 newly identified concerns (C8–C10). No new features — strictly hardening, correctness, and hygiene. - -**Source:** `CODE-EVAL.md` at commit `0f7f8e6` - -**Priority key:** P0 = critical (high severity), P1 = important (medium), P2 = housekeeping (low/negligible). - ---- - -## Task Cards - -### 16.1 — Crypto Adapter Behavioral Normalization *(P0)* — C8 - -**Problem** - -The three CryptoPort adapters (Node, Bun, Web) have inconsistent validation and error-handling behavior — a Liskov Substitution violation. Specifically: - -1. `NodeCryptoAdapter.encryptBuffer()` is synchronous; Bun and Web are async. -2. `BunCryptoAdapter.decryptBuffer()` calls `_validateKey(key)`; Node and Web do not. -3. `NodeCryptoAdapter.createEncryptionStream()` has no premature-finalize guard; Bun and Web throw `CasError('STREAM_NOT_CONSUMED')`. - -Code that works on Bun (early key validation) may produce a cryptic `node:crypto` error on Node. A bug in stream consumption produces undefined behavior on Node but a clear error on Bun/Deno. - -**Fix** - -1. Add `_validateKey(key)` call to `NodeCryptoAdapter.decryptBuffer()` and `WebCryptoAdapter.decryptBuffer()`. -2. Add `streamFinalized` guard + `CasError('STREAM_NOT_CONSUMED')` to `NodeCryptoAdapter.createEncryptionStream()`. -3. Make `NodeCryptoAdapter.encryptBuffer()` explicitly `async` (return `Promise`). -4. Add a cross-adapter behavioral conformance test suite asserting identical behavior for all three adapters given the same inputs. - -**Files:** -- `src/infrastructure/adapters/NodeCryptoAdapter.js` -- `src/infrastructure/adapters/WebCryptoAdapter.js` -- New: `test/unit/infrastructure/adapters/CryptoAdapter.conformance.test.js` - -**Tests:** -```js -describe('16.1: CryptoPort LSP conformance', () => { - // Run the same assertions against all three adapters - for (const [name, adapter] of adapters) { - it(`${name}.encryptBuffer returns a Promise`, ...); - it(`${name}.decryptBuffer rejects invalid key type before crypto error`, ...); - it(`${name}.decryptBuffer rejects wrong-length key before crypto error`, ...); - it(`${name}.createEncryptionStream.finalize() throws STREAM_NOT_CONSUMED if not consumed`, ...); - } -}); -``` - -| Estimate | ~50 LoC changes, ~100 LoC tests, ~4h | -|----------|---------------------------------------| - ---- - -### 16.2 — Memory Restore Guard *(P0)* — C1 - -**Problem** - -`_restoreBuffered()` concatenates ALL chunk blobs into a single buffer before decryption. A 1 GB encrypted file requires ~2 GB of heap. No guard, no warning, no configurable limit. - -**Fix** - -Add `maxRestoreBufferSize` option to CasService constructor (default 512 MiB). Before `Buffer.concat()` in `_restoreBuffered()`, check `manifest.size` against the limit. Throw `CasError('RESTORE_TOO_LARGE')` with an actionable message. - -**Files:** -- `src/domain/services/CasService.js` -- `index.js` (facade wiring) -- `index.d.ts` (type update) - -**Tests:** -```js -describe('16.2: Memory guard on encrypted restore', () => { - it('throws RESTORE_TOO_LARGE when manifest.size exceeds maxRestoreBufferSize', ...); - it('succeeds when manifest.size is within maxRestoreBufferSize', ...); - it('does not apply guard to unencrypted uncompressed restoreStream', ...); - it('includes actionable hint in error message', ...); - it('default maxRestoreBufferSize is 512 MiB', ...); -}); -``` - -| Estimate | ~25 LoC changes, ~40 LoC tests, ~2h | -|----------|--------------------------------------| - ---- - -### 16.3 — Web Crypto Encryption Buffer Guard *(P1)* — C4 - -**Problem** - -`WebCryptoAdapter.createEncryptionStream()` silently buffers the entire stream because Web Crypto AES-GCM is a one-shot API. On Deno, a user calling `store()` with a large encrypted source OOMs without warning. - -**Fix** - -Track accumulated bytes in the `encrypt()` generator. When total exceeds a configurable limit (default 512 MiB), throw `CasError('ENCRYPTION_BUFFER_EXCEEDED')` with an actionable message. - -**Files:** -- `src/infrastructure/adapters/WebCryptoAdapter.js` - -**Tests:** -```js -describe('16.3: Web Crypto buffering guard', () => { - it('throws ENCRYPTION_BUFFER_EXCEEDED when accumulated bytes exceed limit', ...); - it('succeeds for data within buffer limit', ...); - it('NodeCryptoAdapter does NOT throw for large streams (true streaming)', ...); -}); -``` - -| Estimate | ~15 LoC changes, ~30 LoC tests, ~1h | -|----------|--------------------------------------| - ---- - -### 16.4 — FixedChunker Pre-Allocated Buffer *(P2)* — C9 - -**Problem** - -`FixedChunker.chunk()` uses `Buffer.concat([buffer, data])` in a loop. Each call copies the entire accumulated buffer — O(n^2 / chunkSize) total copies for many small input buffers. The CDC chunker uses a pre-allocated working buffer with zero intermediate copies. - -**Fix** - -Replace the concat loop with a pre-allocated `Buffer.allocUnsafe(chunkSize)` working buffer using a copy+offset pattern, matching CdcChunker's approach. - -**Files:** -- `src/infrastructure/chunkers/FixedChunker.js` - -**Tests:** - -Existing tests cover byte-exact correctness. Add: -```js -describe('16.4: FixedChunker buffer efficiency', () => { - it('produces identical output to previous implementation (regression)', ...); - it('handles many small input buffers without excessive allocation', ...); -}); -``` - -| Estimate | ~20 LoC changes, ~15 LoC tests, ~1h | -|----------|--------------------------------------| - ---- - -### 16.5 — Encrypt-Then-Chunk Dedup Warning *(P1)* — C10 - -**Problem** - -Encryption is applied before chunking, destroying content-addressable deduplication. AES-GCM ciphertext is pseudorandom — identical plaintext produces different ciphertext. Users who enable both encryption and CDC chunking get CDC's overhead without its dedup benefit. - -This is an inherent architectural constraint (not fixable without per-chunk encryption). The correct action is documentation + a runtime warning. - -**Fix** - -1. When `store()` is called with both an encryption key/passphrase/recipients AND `chunker.strategy === 'cdc'`, emit `observability.log('warn', 'CDC deduplication is ineffective with encryption — ciphertext is pseudorandom', { strategy: 'cdc' })`. -2. Add a "Known Limitations" section to the README documenting this trade-off. - -**Files:** -- `src/domain/services/CasService.js` (warning in `store()`) - -**Tests:** -```js -describe('16.5: Encrypt-then-chunk dedup warning', () => { - it('emits warning when encryption + CDC chunking are combined', ...); - it('does not warn for encryption + fixed chunking', ...); - it('does not warn for CDC chunking without encryption', ...); -}); -``` - -| Estimate | ~10 LoC changes, ~20 LoC tests, ~1h | -|----------|--------------------------------------| - ---- - -### 16.6 — Chunk Size Upper Bound *(P1)* — C3 - -**Problem** - -`CasService` enforces a minimum chunk size (1024 bytes) but no maximum. A user can configure a 4 GB chunk size. Additionally, `FixedChunker` and `CdcChunker` accept arbitrarily large values without validation. - -**Fix** - -1. Add `if (chunkSize > MAX_CHUNK_SIZE)` guard in `CasService` constructor. 100 MiB is the cap — generous while staying within Git hosting limits. -2. Emit `observability.log('warn', ...)` when chunkSize exceeds 10 MiB. -3. Add matching validation in `FixedChunker` constructor: `if (chunkSize > 100 * 1024 * 1024) throw new RangeError(...)`. -4. Add matching validation in `CdcChunker` constructor for `maxChunkSize`. - -**Files:** -- `src/domain/services/CasService.js` -- `src/infrastructure/chunkers/FixedChunker.js` -- `src/infrastructure/chunkers/CdcChunker.js` - -**Tests:** -```js -describe('16.6: Chunk size upper bound', () => { - it('CasService throws when chunkSize exceeds 100 MiB', ...); - it('CasService accepts chunkSize of exactly 100 MiB', ...); - it('FixedChunker throws when chunkSize exceeds 100 MiB', ...); - it('CdcChunker throws when maxChunkSize exceeds 100 MiB', ...); - it('logs warning when chunkSize exceeds 10 MiB', ...); -}); -``` - -| Estimate | ~15 LoC changes, ~30 LoC tests, ~1h | -|----------|--------------------------------------| - ---- - -### 16.7 — Lifecycle Method Naming *(P2)* - -**Problem** - -`deleteAsset()` does not delete anything — it reads a manifest and returns metadata about what would be orphaned. `findOrphanedChunks()` doesn't find orphans — it collects referenced chunk OIDs. Both names are misleading. - -**Fix** - -1. Add `inspectAsset({ treeOid })` as the canonical name. `deleteAsset` becomes a deprecated alias that delegates to `inspectAsset`. -2. Add `collectReferencedChunks({ treeOids })` as the canonical name. `findOrphanedChunks` becomes a deprecated alias. -3. Emit `observability.log('warn', 'deleteAsset() is deprecated — use inspectAsset()')` on deprecated path. -4. Update `index.d.ts` with `@deprecated` JSDoc on old methods. - -This is a **non-breaking** deprecation. Removal is deferred to a future major version. - -**Files:** -- `src/domain/services/CasService.js` -- `index.js` (facade) -- `index.d.ts` - -**Tests:** -```js -describe('16.7: Lifecycle method naming', () => { - it('inspectAsset returns { slug, chunksOrphaned }', ...); - it('deleteAsset delegates to inspectAsset (deprecated alias)', ...); - it('collectReferencedChunks returns { referenced, total }', ...); - it('findOrphanedChunks delegates to collectReferencedChunks (deprecated alias)', ...); -}); -``` - -| Estimate | ~30 LoC changes, ~25 LoC tests, ~1h | -|----------|--------------------------------------| - ---- - -### 16.8 — CasError Portability Guard *(P2)* - -**Problem** - -`CasError` calls `Error.captureStackTrace(this, this.constructor)` unconditionally. This is V8-specific — it's a no-op on Bun's JavaScriptCore engine. While it doesn't crash (JSC silently ignores it), it indicates incomplete multi-runtime awareness. - -**Fix** - -Guard the call: `if (Error.captureStackTrace) Error.captureStackTrace(this, this.constructor);` - -**Files:** -- `src/domain/errors/CasError.js` - -**Tests:** -```js -describe('16.8: CasError multi-runtime portability', () => { - it('creates CasError with code and meta', ...); - it('does not throw when Error.captureStackTrace is unavailable', ...); -}); -``` - -| Estimate | ~3 LoC changes, ~10 LoC tests, ~0.5h | -|----------|---------------------------------------| - ---- - -### 16.9 — Pre-Commit Hook + Hooks Directory *(P2)* - -**Problem** - -The project has a `pre-push` hook but no `pre-commit` hook. Lint failures are not caught until push time. Additionally, the hooks directory is `scripts/git-hooks/` rather than `scripts/hooks/` per the CLAUDE.md convention. - -**Fix** - -1. Rename `scripts/git-hooks/` to `scripts/hooks/`. -2. Update `scripts/install-hooks.sh` to reference the new path. -3. Add `scripts/hooks/pre-commit` that runs `pnpm run lint`. -4. Update `.git/config` hooksPath if already set. - -**Files:** -- `scripts/git-hooks/pre-push` → `scripts/hooks/pre-push` -- New: `scripts/hooks/pre-commit` -- `scripts/install-hooks.sh` - -| Estimate | ~15 LoC, ~0.5h | -|----------|-----------------| - ---- - -### 16.10 — Orphaned Blob Tracking *(P1)* — C2 - -**Problem** - -When `_chunkAndStore()` throws `STREAM_ERROR`, chunks already written to Git are orphaned. The error meta reports `chunksDispatched` but not the blob OIDs of successful writes. There's no visibility into what was orphaned. - -**Fix** - -1. After `Promise.allSettled(pending)`, collect blob OIDs from fulfilled results. -2. Include `orphanedBlobs: string[]` in the `STREAM_ERROR` meta. -3. Emit `observability.metric('error', { action: 'orphaned_blobs', count, blobs })`. - -**Files:** -- `src/domain/services/CasService.js` - -**Tests:** -```js -describe('16.10: Orphaned blob tracking on STREAM_ERROR', () => { - it('includes orphanedBlobs array in STREAM_ERROR meta', ...); - it('orphanedBlobs contains blob OIDs from successful writes before failure', ...); - it('orphanedBlobs is empty when stream fails before any writes', ...); - it('emits orphaned_blobs metric via observability', ...); -}); -``` - -| Estimate | ~20 LoC changes, ~30 LoC tests, ~2h | -|----------|--------------------------------------| - ---- - -### 16.11 — Passphrase Input Security *(P0)* — C5 + V6 - -**Problem** - -`--vault-passphrase ` puts the passphrase in shell history and process listings. The `GIT_CAS_PASSPHRASE` env var is better but still visible in `/proc//environ`. - -**Fix** - -1. **Interactive prompt**: When `--vault-passphrase` is passed without a value and stdin is a TTY, prompt with echo disabled. Confirmation on first use (store/init). -2. **File-based input**: Add `--vault-passphrase-file ` flag that reads from a file. -3. **Stdin pipe**: `--vault-passphrase -` reads from stdin. -4. **Documentation**: Security warning in `--help` and README. - -**Files:** -- `bin/git-cas.js` -- New: `bin/ui/passphrase-prompt.js` - -**Tests:** -```js -describe('16.11: Passphrase input security', () => { - it('reads passphrase from file when --vault-passphrase-file is used', ...); - it('errors when no passphrase source is available in non-TTY mode', ...); - it('--vault-passphrase-file trims trailing newline', ...); -}); -``` - -| Estimate | ~90 LoC, ~30 LoC tests, ~4h | -|----------|------------------------------| - ---- - -### 16.12 — KDF Brute-Force Awareness *(P2)* — C6 - -**Problem** - -`deriveKey()` and the restore path have no rate limiting or audit trail. An attacker can brute-force passphrases at full CPU speed. - -**Fix** - -1. Emit `observability.metric('error', { action: 'decryption_failed', slug })` on every `INTEGRITY_ERROR` during passphrase-based restore. -2. In the CLI layer, add a 1-second delay after each failed passphrase attempt. - -**Files:** -- `src/domain/services/CasService.js` (observability metric) -- `bin/git-cas.js` (CLI delay) - -**Tests:** -```js -describe('16.12: KDF brute-force awareness', () => { - it('emits decryption_failed metric on wrong passphrase', ...); - it('emits metric with slug context for audit trail', ...); - it('library API does NOT rate-limit (callers manage their own policy)', ...); -}); -``` - -| Estimate | ~10 LoC changes, ~20 LoC tests, ~1h | -|----------|--------------------------------------| - ---- - -### 16.13 — GCM Nonce Collision Documentation *(P2)* — C7 - -**Problem** - -AES-256-GCM uses a 96-bit random nonce. Birthday bound is ~2^48; NIST recommends limiting to 2^32 invocations per key. There's no tracking, no warning, and no documentation of the bound. - -**Fix** - -1. Add `SECURITY.md` at project root documenting: GCM nonce bound, recommended key rotation frequency, KDF parameter guidance, passphrase entropy recommendations. -2. Add `encryptionCount` field to vault metadata. Increment per `store()` with encryption. Emit observability warning when count exceeds 2^31. - -**Files:** -- New: `SECURITY.md` -- `src/domain/services/VaultService.js` (counter increment) - -**Tests:** -```js -describe('16.13: Nonce usage tracking', () => { - it('vault metadata includes encryptionCount after encrypted store', ...); - it('encryptionCount increments per encrypted store', ...); - it('warns via observability when encryptionCount exceeds threshold', ...); -}); -``` - -| Estimate | ~25 LoC changes, ~20 LoC tests, ~2h | -|----------|--------------------------------------| - ---- - -### M16 Summary - -| Task | Theme | Priority | Severity | Audit Ref | Concern Ref | ~LoC | ~Hours | -|------|-------|----------|----------|-----------|-------------|------|--------| -| 16.1 | Crypto adapter normalization | P0 | High | Flaw 1 | C8 | ~150 | ~4h | -| 16.2 | Memory restore guard | P0 | High | Flaw 2 | C1 | ~65 | ~2h | -| 16.3 | Web Crypto buffer guard | P1 | Medium | Flaw 3 | C4 | ~45 | ~1h | -| 16.4 | FixedChunker buffer optimization | P2 | Low | Flaw 4 | C9 | ~35 | ~1h | -| 16.5 | Encrypt-then-chunk dedup warning | P1 | Medium | Flaw 5 | C10 | ~30 | ~1h | -| 16.6 | Chunk size upper bound | P1 | Medium | Flaw 6 | C3 | ~45 | ~1h | -| 16.7 | Lifecycle method naming | P2 | Low | Flaw 7 | — | ~55 | ~1h | -| 16.8 | CasError portability guard | P2 | Negligible | Flaw 8 | — | ~13 | ~0.5h | -| 16.9 | Pre-commit hook + hooks dir | P2 | Low | Flaw 9 | — | ~15 | ~0.5h | -| 16.10 | Orphaned blob tracking | P1 | Medium | — | C2 | ~50 | ~2h | -| 16.11 | Passphrase input security | P0 | High | — | C5+V6 | ~120 | ~4h | -| 16.12 | KDF brute-force awareness | P2 | Low | — | C6 | ~30 | ~1h | -| 16.13 | GCM nonce collision docs + counter | P2 | Low | — | C7 | ~45 | ~2h | -| **Total** | | | | | | **~698** | **~21h** | - -### Recommended Execution Order - -**Phase 1 — Safety nets (P0):** -16.8, 16.9, 16.1, 16.2, 16.11 - -**Phase 2 — Correctness (P1):** -16.6, 16.3, 16.5, 16.10 - -**Phase 3 — Polish (P2):** -16.4, 16.7, 16.12, 16.13 - ---- - -# 7) Feature Matrix - -Competitive landscape for content-addressed storage, encrypted binary assets, and large-file Git tooling. Rows represent the union of features across the space — not just what git-cas offers, but what users encounter and expect when evaluating tools in this category. - -**Legend:** ✅ Yes | ⚠️ Partial | ❌ No | 🗓 Planned | N/A Not applicable - -**Competitors:** -- **Git LFS** — Large file storage via external server + pointer files -- **git-annex** — Distributed file management with GPG encryption and location tracking -- **Restic** — Encrypted backup with CDC dedup -- **Age** — Modern file encryption primitive (not a storage system) -- **DVC** — Data/ML version control with multi-backend remotes - ---- - -### Storage & Chunking - -| Feature | git-cas v2.0 | Planned | Git LFS | git-annex | Restic | Age | DVC | Use Case | Remarks | What it would take | -|---|---|---|---|---|---|---|---|---|---|---| -| Content-addressed storage | ✅ SHA-256 | — | ✅ SHA-256 | ✅ SHA-256/512 | ✅ SHA-256 | ❌ | ✅ MD5 | Dedup, integrity, immutability | git-cas is Git-native; others use separate object stores | — | -| Fixed-size chunking | ✅ 256 KiB default, configurable | — | ❌ | ⚠️ Special remotes only | ❌ | ❌ | ❌ | Break large files into stable blobs | Simple and deterministic; poor dedup on edits | — | -| Content-defined chunking (CDC) | ✅ v5.0.0 Buzhash | — | ❌ | ❌ | ✅ Rabin fingerprint, 512K–8M | ❌ | ❌ | Sub-file dedup on versioned data | Buzhash CDC engine with 98% chunk reuse on small edits | — | -| Sub-file deduplication | ✅ Via chunking | ✅ Via CDC | ❌ | ⚠️ Chunk-level only | ✅ Via CDC | ❌ | ❌ | Avoid storing redundant bytes | Fixed chunks dedup exact matches; CDC handles shifted content | CDC (M10) improves from exact-match to shift-tolerant | -| File-level deduplication | ✅ Git ODB | — | ✅ | ✅ | ✅ | ❌ | ✅ | Identical files stored once | All CAS systems get this for free | — | -| Git-native storage (ODB) | ✅ Blobs + trees | — | ❌ Separate LFS store | ⚠️ Pointers in ODB, content in annex | ❌ Custom format | ❌ | ❌ Cache dir | Inspectable via `git log`, replicable via `git push` | Unique to git-cas. Competitors use custom storage layers | — | -| External server required | ❌ | — | ✅ LFS server | ❌ | ❌ | ❌ | ❌ | Self-contained local operation | git-cas and git-annex work fully offline. LFS requires server infra | — | - ---- - -### Encryption & Key Management - -| Feature | git-cas v2.0 | Planned | Git LFS | git-annex | Restic | Age | DVC | Use Case | Remarks | What it would take | -|---|---|---|---|---|---|---|---|---|---|---| -| Client-side encryption | ✅ AES-256-GCM | — | ❌ | ✅ GPG | ✅ AES-256-CTR + Poly1305 | ✅ ChaCha20-Poly1305 | ❌ | Protect data at rest in untrusted storage | git-cas is the only Git-native tool with integrated encryption | — | -| Authenticated encryption (AEAD) | ✅ GCM auth tag | — | ❌ | ⚠️ GPG signature optional | ✅ Poly1305 | ✅ Poly1305 | ❌ | Tamper detection + confidentiality | GCM and Poly1305 both provide authentication. GPG can but doesn't by default | — | -| Per-chunk encryption | ✅ Streaming | — | ❌ | ❌ Whole-file | ❌ Per-pack | ✅ 64 KiB chunks | ❌ | Encrypt without buffering full file | git-cas and Age both stream; Restic encrypts packed blobs | — | -| Multi-recipient encryption | ✅ M11 Locksmith | — | ❌ | ✅ Multiple GPG keys | ✅ Multiple passwords | ✅ Multiple X25519 | ❌ | Team access without sharing a single key | Envelope encryption (DEK/KEK model) | — | -| Key rotation (no re-encrypt) | ✅ M12 Carousel | — | N/A | ⚠️ Can add keys; revoke requires re-encrypt | ✅ Re-wrap master key | ❌ | N/A | Respond to key compromise without re-storing data | Re-wraps DEK, data blobs untouched | — | -| KDF / passphrase keys | ✅ PBKDF2, scrypt | — | ❌ | ✅ GPG S2K | ✅ scrypt | ✅ scrypt | ❌ | Derive keys from passwords instead of managing raw bytes | git-cas supports both PBKDF2 (100k iterations) and scrypt | — | -| Argon2 KDF | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | Memory-hard KDF resists GPU/ASIC attacks | No tool in this space supports Argon2 yet. Would require native/WASM addon | ~80 LoC + native dep. ~4h. Low priority — scrypt is adequate | -| Hardware security (YubiKey/HSM) | ❌ | ❌ | ❌ | ✅ GPG smartcard | ❌ | ✅ age-plugin-yubikey | ❌ | Keys never leave hardware token | Would require plugin system or GPG integration | Plugin architecture + PIV applet integration. ~300 LoC, ~16h. Low priority | -| SSH key as encryption identity | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ ed25519/RSA | ❌ | Encrypt to existing SSH keys without new key material | Age's signature feature; niche but convenient | X25519 key derivation from SSH ed25519. ~150 LoC, ~8h. Low priority | - ---- - -### Compression & I/O - -| Feature | git-cas v2.0 | Planned | Git LFS | git-annex | Restic | Age | DVC | Use Case | Remarks | What it would take | -|---|---|---|---|---|---|---|---|---|---|---| -| Compression | ✅ gzip | — | ❌ | ⚠️ Via GPG (zlib/bzip2) | ✅ zstandard | ❌ | ❌ | Reduce storage size for compressible data | Compress-before-encrypt pipeline. Only git-cas and Restic offer explicit control | — | -| Compression algorithm selection | ❌ gzip only | ❌ | ❌ | ⚠️ GPG's choice | ✅ zstd auto/max/off | ❌ | ❌ | Tune speed vs. ratio per workload | zstd is faster + better ratio than gzip. Would need CompressionPort | CompressionPort + zstd adapter. ~120 LoC, ~6h. Medium priority | -| Streaming store (O(1) memory) | ✅ AsyncIterable | — | ⚠️ Transfer adapters | ✅ GPG pipeline | ✅ Pack streaming | ✅ 64 KiB chunks | ❌ | Store arbitrarily large files without OOM | git-cas chunks and encrypts in streaming fashion | — | -| Streaming restore (O(1) memory) | ✅ restoreStream() | — | ⚠️ | ✅ | ✅ | ✅ | ❌ | Restore large files without OOM | Implemented in v4.0.0 (M14 Conduit) | — | -| Partial restore / byte-range | ❌ | ❌ | ❌ | ⚠️ Per-chunk retrieval | ✅ FUSE mount | ❌ | ❌ | Extract byte ranges without restoring full file | Manifest has chunk offsets; byte-range index is feasible | Chunk offset index + range API. ~200 LoC, ~10h. Low priority | - ---- - -### Manifests & Indexing - -| Feature | git-cas v2.0 | Planned | Git LFS | git-annex | Restic | Age | DVC | Use Case | Remarks | What it would take | -|---|---|---|---|---|---|---|---|---|---|---| -| Manifest / index format | ✅ JSON or CBOR | — | Pointer files (text) | Symlinks + location log | JSON index (encrypted) | Binary header | YAML .dvc files | Describe stored assets for retrieval | git-cas is unique in offering codec choice (JSON for humans, CBOR for perf) | — | -| Codec pluggability | ✅ JsonCodec, CborCodec | — | ❌ | ❌ | ❌ | ❌ | ❌ | Choose manifest format per use case | Extensible via CodecPort. No other tool offers this | — | -| Merkle tree manifests | ✅ v2 auto-split | — | ❌ | ❌ | ❌ | ❌ | ❌ | Scale manifests for millions of chunks | Auto-splits at threshold (default 1000). Transparent reconstitution | — | -| Vault / ref-based indexing | ✅ refs/cas/vault | — | ❌ | ✅ git-annex branch | ❌ | ❌ | ❌ | GC-safe asset index that survives `git gc` | CAS semantics with retry. Unique among Git-native tools | — | -| Manifest versioning | ✅ v1 flat, v2 Merkle + chunking | — | Pointer v1 only | ❌ | ❌ | ❌ | ❌ | Evolve format without breaking old manifests | Full backward compat: v2 code reads v1 manifests | — | - ---- - -### Lifecycle & Management - -| Feature | git-cas v2.0 | Planned | Git LFS | git-annex | Restic | Age | DVC | Use Case | Remarks | What it would take | -|---|---|---|---|---|---|---|---|---|---|---| -| Integrity verification | ✅ SHA-256 + GCM tag | — | ✅ SHA-256 | ✅ `annex fsck` | ✅ `restic check` | ✅ Poly1305 | ✅ MD5/SHA256 | Detect corruption or tampering | Per-chunk digest + auth tag. `verifyIntegrity()` API | — | -| Garbage collection | ⚠️ Vault prevents GC loss; manual `git gc` for cleanup | — | ✅ `lfs prune` | ✅ `unused` + `dropunused` | ✅ `forget` + `prune` | N/A | ✅ `dvc gc` | Reclaim storage from deleted assets | Vault refs keep blobs reachable. No automated sweeper | Vault squash + storage stats. ~80 LoC, ~3h. Low priority | -| Lifecycle management | ✅ readManifest, deleteAsset, findOrphanedChunks | — | ⚠️ Prune + server policies | ✅ Full (unused, drop, dead, whereis, numcopies) | ✅ Retention policies | N/A | ⚠️ `dvc gc` with scope flags | Inspect, audit, and plan deletions | git-annex is most mature. git-cas provides the primitives | — | -| Retention policies (time/count) | ❌ | ❌ | ❌ | ❌ | ✅ keep-last, keep-daily, keep-weekly, etc. | N/A | ❌ | Automated pruning by age or count | Backup-oriented feature. Out of scope for CAS library | Policy engine + vault history scanning. ~200 LoC, ~8h. Not planned | -| Incremental backups / snapshots | ❌ | ❌ | N/A | ✅ Sync transfers only changed content | ✅ Core design | N/A | ✅ Only changed files pushed | Efficient repeated backups | git-cas stores individual assets, not snapshot trees | Snapshot tree structure + diff engine. ~400 LoC, ~20h. Not planned | -| Location tracking | ❌ | ❌ | ❌ | ✅ `whereis`, numcopies, trust levels | ❌ | N/A | ❌ | Know which remotes hold copies of each file | git-annex's defining feature. Orthogonal to CAS | Location log in vault metadata. ~250 LoC, ~12h. Not planned | -| FUSE mount | ❌ | ❌ | ❌ | ⚠️ Third-party | ✅ `restic mount` | ⚠️ Rust `rage` only | ❌ | Browse stored assets as a filesystem | Requires platform-specific FUSE bindings | libfuse binding + virtual FS. ~500 LoC, ~24h. Not planned | - ---- - -### Observability & Developer Experience - -| Feature | git-cas v2.0 | Planned | Git LFS | git-annex | Restic | Age | DVC | Use Case | Remarks | What it would take | -|---|---|---|---|---|---|---|---|---|---|---| -| CLI tool | ✅ `git cas` subcommand | — | ✅ `git lfs` | ✅ `git annex` | ✅ `restic` | ✅ `age` | ✅ `dvc` | Terminal-based workflows | All tools have CLIs. git-cas integrates as a Git subcommand | — | -| Programmatic API / library | ✅ Node.js (ESM) | — | ⚠️ Go internal | ⚠️ Haskell | ⚠️ Go internal | ✅ Go, Rust, JS, Java, Python | ✅ Python | Integrate CAS into applications | git-cas and Age are the strongest library stories | — | -| Multi-runtime support | ✅ Node, Bun, Deno | — | ❌ Go only | ❌ Haskell only | ❌ Go only | ✅ Go, Rust, JS, Java, Python | ❌ Python only | Same library works across JS runtimes | Only git-cas and Age support multiple runtimes | — | -| Progress events (structured) | ✅ ObservabilityPort (metric/log/span) | — | ✅ Transfer protocol | ⚠️ Terminal bars | ✅ JSON Lines | ❌ | ⚠️ Terminal bars | Build progress bars, logging, monitoring | git-cas emits typed metrics per chunk via ObservabilityPort (v4.0.0) | — | -| CLI progress feedback | ✅ Animated (bijou) | — | ✅ | ✅ | ✅ | ❌ | ✅ | Users know operations are working | Implemented in v3.1.0 (M13 Bijou) | — | -| Structured output (--json) | ✅ `--json` | — | ❌ | ❌ | ✅ `--json` | ❌ | ✅ `--json` | CI/CD pipeline integration | Global `--json` flag on all commands | — | -| CLI `verify` command | ✅ `git cas verify` | — | ✅ Implicit on checkout | ✅ `annex fsck` | ✅ `restic check` | ❌ | ✅ `dvc check-ignore` | Audit integrity without restoring | Per-chunk SHA-256 verification | — | -| Actionable error messages | ✅ Hints | — | ⚠️ | ⚠️ | ✅ | ❌ | ✅ | Users know what went wrong and what to do next | Error codes + actionable hint map | — | - ---- - -### Integration & Ecosystem - -| Feature | git-cas v2.0 | Planned | Git LFS | git-annex | Restic | Age | DVC | Use Case | Remarks | What it would take | -|---|---|---|---|---|---|---|---|---|---|---| -| Multi-backend storage (S3, etc.) | ❌ Git ODB only | ❌ | ⚠️ Custom transfer adapters | ✅ S3, rsync, WebDAV, IPFS, bittorrent, rclone, etc. | ✅ S3, SFTP, Azure, GCS, Swift | N/A | ✅ S3, Azure, GCS, HDFS, SSH | Store content on cloud/remote infrastructure | git-cas deliberately uses Git as the transport layer (push/pull) | Remote backend port. ~300 LoC, ~16h. Not planned — Git remotes serve this role | -| File locking (pessimistic) | ❌ | ❌ | ✅ Lock API | ❌ | N/A | N/A | ❌ | Prevent concurrent edits on binary files | LFS-specific feature for team workflows on unmergeable files | Lock API on vault entries. ~150 LoC, ~8h. Not planned | -| Plugin / extension system | ❌ Ports (compile-time) | ❌ | ✅ Transfer adapters | ✅ External special remotes | ❌ | ✅ age-plugin-* | ✅ Remote plugins | Extend with custom backends, crypto, etc. | git-cas uses ports/adapters pattern (hexagonal), but no runtime plugin loading | Runtime plugin discovery. ~200 LoC, ~10h. Not planned | -| ML experiment tracking | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Metrics, params, plots, DVCLive | Track ML experiments with data versioning | DVC's differentiator. Out of scope for a CAS library | N/A — different product category | -| Pipeline / DAG execution | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ `dvc.yaml` + `dvc repro` | Reproducible data processing pipelines | DVC-specific. Out of scope | N/A — different product category | -| Post-quantum cryptography | ❌ | ❌ | ❌ | ❌ | ❌ | 🗓 X-Wing hybrid | ❌ | Future-proof against quantum attacks | Age has this on its roadmap (X-Wing KEM). Very early-stage across the industry | X-Wing KEM integration. Research-stage. Not planned | - ---- - -### Competitive Summary - -| | git-cas | Git LFS | git-annex | Restic | Age | DVC | -|---|---|---|---|---|---|---| -| **Core identity** | Git-native CAS with encryption | Git large file offloading | Distributed file management | Encrypted backup with dedup | File encryption primitive | ML data version control | -| **Strongest at** | Git ODB integration, pluggable codecs, Merkle manifests, vault | Simplicity, file locking, ecosystem adoption | Backend diversity, location tracking, metadata views | CDC dedup, retention policies, FUSE mount | Multi-recipient, HSM, multi-language, simplicity | ML pipelines, experiment tracking, Python ecosystem | -| **Weakest at** | No multi-backend, gzip only | No encryption, no compression, requires server | Complexity, Haskell-only, no CDC | No Git integration, no library API | Not a storage system | No encryption, no chunking, no streaming | -| **Server required** | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| **Best use case** | Encrypted binary assets in Git repos | Large files in GitHub/GitLab repos | Distributed archive management | Encrypted backups of filesystems | Encrypting files for recipients | ML model/data versioning | - ---- - -### Where git-cas leads - -1. **Git-native CAS** — Only tool that stores content directly in Git's object database. Assets are inspectable via `git log`, replicable via `git push`, and addressable via tree OIDs. No custom binary format, no external server. -2. **Merkle tree manifests** — No competitor offers automatic manifest splitting for very large files. -3. **Codec pluggability** — JSON for human readability, CBOR for binary efficiency. No other tool lets you choose. -4. **Vault with CAS semantics** — Atomic ref-based indexing with conflict detection and retry. Assets survive `git gc`. -5. **Multi-runtime JS library** — Works on Node, Bun, and Deno. Only Age offers comparable multi-runtime coverage. - -### Where git-cas trails (and what closes the gap) - -1. **Multi-recipient encryption** → M11 Locksmith. DEK/KEK envelope encryption. ~580 LoC, ~20h. -2. **Content-defined chunking** → M10 Hydra. Buzhash CDC engine + ChunkingPort. ~690 LoC, ~22h. -3. **Key rotation** → M12 Carousel. Re-wrap DEK without re-encrypting data. ~400 LoC, ~13h. -4. ~~**Streaming restore**~~ → ✅ Delivered in v4.0.0 (M14 Conduit). `restoreStream()` returning AsyncIterable. -5. **CLI polish** → M9 Cockpit. Verify, --json, actionable errors. ~190 LoC, ~5h. -6. ~~**CLI progress feedback**~~ → ✅ Delivered in v3.1.0 (M13 Bijou). Animated progress bars with throughput. -7. **Multi-backend storage** → Not planned. Git remotes serve as the transport layer by design. Adding S3/SFTP backends would dilute the "Git-native" identity. -8. **Compression algorithm selection** → Not on roadmap. CompressionPort + zstd adapter would cost ~120 LoC, ~6h. Medium priority. -9. **FUSE mount / partial restore** → Not planned. Niche for a CAS library. Would require ~500 LoC + platform-specific bindings. - ---- - -# 8) Competitive Analysis — When to Use git-cas (and When Not To) - -## The one-line pitch - -git-cas is for people who want Git to be the whole stack — object store, transport, access control, audit log — and don't want to bolt on a second system for binary assets. - -Every competitor in this space either requires an external server (Git LFS), invents a custom storage format (Restic, DVC), or isn't a storage system at all (Age). git-cas is the only tool that writes content directly into Git's object database as native blobs and trees, meaning your assets travel with `git push`, deduplicate with `git gc`, and are addressable with tree OIDs in commits and tags. If that sentence excited you, this is your tool. If it didn't, keep reading — one of the others might be a better fit. - ---- - -## When to use git-cas - -### 1. Encrypted binary assets in a Git monorepo - -**Scenario:** You're building a game, a design system, or a firmware project. You have binary assets (textures, fonts, model weights, firmware images) that belong in the same repo as your source code. Some of them are sensitive (signing keys, license bundles, proprietary models) and need to be encrypted at rest, even in private repos. - -**Why git-cas:** Assets live in the Git ODB alongside your source. They're committed, branched, tagged, and pushed like any other object. Encryption is AES-256-GCM, integrated into the chunking pipeline, not bolted on after the fact. The vault keeps them GC-safe. You don't need a second storage system, a second credential set, or a second billing account. - -**Why not Git LFS:** LFS can't encrypt. LFS requires a server. LFS pointer files are not the content — they're redirects to an external store that you have to provision, pay for, and maintain separately. - -**Why not git-annex:** git-annex can do this, but it stores content in `.git/annex/objects`, not in the Git ODB. It requires GPG for encryption (heavyweight, config-heavy, S2K-based KDF). It's a Haskell binary — you can't import it as a library in your Node/TypeScript build system. If you're already in the Haskell ecosystem and need distributed location tracking, git-annex is phenomenal. If you're in the JavaScript ecosystem and want a library, it's the wrong tool. - ---- - -### 2. Self-hosted secret bundles without external infrastructure - -**Scenario:** Your team stores deployment secrets, TLS certificates, or environment bundles in a private Git repo. You want encryption at rest, passphrase-based access, and zero dependency on external services (no Vault server, no AWS KMS, no 1Password CLI). - -**Why git-cas:** `git cas store ./secrets.tar.gz --slug prod-secrets --tree --vault-passphrase "correct horse battery staple"` — done. Encrypted, vaulted, GC-safe, and replicable to any Git remote. Restore with the passphrase. No infrastructure. No SaaS. No tokens to rotate (until M12 ships, and then you can rotate those too). - -**Why not sops/Age:** If your secrets are structured YAML/JSON (Kubernetes secrets, Terraform vars), sops is purpose-built for that — it encrypts individual values within the file, so you can `git diff` the structure without decrypting. git-cas encrypts the entire blob. If you need per-field encryption and diffable ciphertext, use sops. If you need to store opaque binary bundles (tarballs, keystores, firmware signing keys), git-cas is the better fit. - ---- - -### 3. Deterministic, content-addressed artifact storage - -**Scenario:** Your CI pipeline produces build artifacts (WASM bundles, compiled binaries, ML model checkpoints). You want to store them content-addressed so identical builds don't duplicate storage, and you want to reference them by tree OID in release commits. - -**Why git-cas:** Store the artifact, get a tree OID, commit that OID in your release tag. The artifact is now permanently addressable at that commit. `git cas restore --oid --out ./artifact.wasm` retrieves it anywhere the repo is cloned. Deduplication is free — if two builds produce identical output, Git stores one copy. Manifests give you a chunk-level inventory with SHA-256 digests. - -**Why not DVC:** If your artifacts are outputs of a reproducible pipeline with parameters, metrics, and experiments, DVC is built exactly for that. DVC tracks inputs → outputs through a DAG, supports experiment comparison, and integrates with ML frameworks via DVCLive. git-cas stores blobs — it doesn't understand pipelines, parameters, or metrics. If you need `dvc repro` and `dvc exp`, use DVC. If you need a dumb content-addressed blob store that lives inside Git, use git-cas. - ---- - -### 4. Embedding binary data packs in libraries or SDKs - -**Scenario:** You're shipping an npm package or JSR module that needs to bundle a data file (a wasm binary, a trained model, a lookup table) that's too large for Git's comfort zone but too tightly coupled to the code to live in a separate system. - -**Why git-cas:** Store the data pack via the programmatic API (`cas.storeFile()`), commit the tree OID, and restore it in your build script or at runtime. The data travels with `git clone` — no post-install fetch from a CDN, no `git lfs pull`, no separate authentication. Your consumers don't need to know git-cas exists; they just clone and build. - -**Why not Git LFS:** LFS requires consumers to have LFS installed and configured, and it requires a server to host the objects. If your package is on npm or JSR, the LFS objects don't travel with `npm install` — they're left behind on the LFS server, which your consumers may not have access to. - ---- - -### 5. Offline-first or air-gapped environments - -**Scenario:** You're working in a classified environment, an air-gapped network, or a submarine (it happens). You need encrypted binary asset management with zero network dependencies. - -**Why git-cas:** Everything is local. `git init`, `npm install @git-stunts/git-cas`, and go. No server, no cloud, no tokens, no DNS resolution. Push to a USB drive via `git bundle` if you need to transfer. Encryption is client-side. The vault is a Git ref. The entire system fits in a single repo directory. - -**Why not anything cloud-dependent:** Git LFS needs a server. DVC's value proposition is built around remote storage (S3, GCS). Restic can work locally but is designed around the backup-to-remote workflow. git-annex is the closest competitor here — it also works fully offline — but it brings GPG complexity and doesn't integrate as a JavaScript library. - ---- - -## When NOT to use git-cas - -### 0. You just want images or demos in your README - -**Use instead: an orphan branch** - -**Scenario:** You want to put screenshots, demo GIFs, and logos in your repo's README. The assets are public, small (< 5 MB each), and you want GitHub to render them inline. - -**Why not git-cas:** It's overkill. You don't need encryption, chunking, manifests, or a vault for a 200 KiB screenshot. git-cas adds a dependency, a CLI workflow, and conceptual overhead for a problem that's already solved by 5 git commands. +## Current Reality -**Why an orphan branch:** `git checkout --orphan assets`, `git rm -rf .`, add your images, commit, push. Reference them with `![Demo](../assets/demo.gif?raw=true)`. No dependencies, no tooling, no build step. GitHub renders them directly. Every developer on your team already knows how to do this. The approach is documented everywhere and works with every Git host. +- **Current release:** `v5.3.1` (2026-03-15) +- **Current line:** M16 Capstone shipped in `v5.3.0`; `v5.3.1` is the maintenance follow-up that fixed repeated-chunk tree emission for repetitive content. +- **Supported runtimes:** Node.js 22.x (primary), Bun, Deno +- **Current operator experience:** the human-facing CLI/TUI is shipped now; the machine-facing agent CLI is planned next. -**The honest line:** If your assets are public and small, the orphan branch pattern is simpler and better. git-cas earns its keep when you need encryption, dedup, compression, integrity verification, or lifecycle management — not when you need a GIF in a README. +## Interface Strategy ---- +`git-cas` now has an explicit two-surface direction: -### 1. You need to back up an entire filesystem +### Human CLI/TUI -**Use instead: Restic** +This is the current public operator surface. -**Scenario:** You want nightly encrypted backups of `/home` or a database dump directory, with 30-day retention, incremental snapshots, and the ability to mount old snapshots as a virtual filesystem. +- Existing `git cas ...` commands remain the stable human workflow. +- Bijou formatting, prompts, dashboards, and TTY-aware behavior stay here. +- `--json` remains supported as convenience structured output for humans and simple scripts. +- Human-facing improvements continue under the Bijou/TUI roadmap. -**Why not git-cas:** git-cas stores individual assets by slug. It doesn't have a concept of filesystem snapshots, retention policies, or incremental diffing. You'd have to build all of that yourself. It doesn't have FUSE mounting. It doesn't have `--keep-daily 7 --keep-weekly 4 --keep-monthly 12`. +### Agent CLI -**Why Restic:** Restic was built for exactly this. Content-defined chunking means incremental backups only store changed chunks. Retention policies automate pruning. `restic mount` lets you browse any snapshot. AES-256 encryption is mandatory and always-on. scrypt KDF. JSON progress output. It's the gold standard for encrypted backups. +This is planned work starting in **M18 Relay**. ---- +- Namespace: `git cas agent` +- Output: JSONL on `stdout` only, one record per line +- No Bijou formatting, no TTY-mode branching, no implicit prompts +- Stable event envelope: `protocol`, `command`, `type`, `seq`, `ts`, `data` +- Reserved record types: `start`, `progress`, `warning`, `needs-input`, `result`, `error`, `end` +- One-shot commands in v1: stream records during execution, then exit +- Non-interactive secret/input handling: + - missing required input -> emit `needs-input`, exit `2` + - fatal execution failure -> emit `error`, exit `1` + - integrity/verification failure -> exit `3` + - success -> exit `0` +- Request input supports normal flags plus `--request -` and `--request @file.json` -### 2. You need large file storage on GitHub/GitLab with team workflows +The agent CLI is a first-class workflow, not an extension of the human `--json` mode. -**Use instead: Git LFS** +## Shipped Summary -**Scenario:** Your team of 30 designers commits Photoshop files, video assets, and 3D models to a GitHub repo. You need file locking so two people don't edit the same binary simultaneously, and you want GitHub's UI to show file sizes and download links. - -**Why not git-cas:** git-cas has no file locking. It has no integration with GitHub's LFS API, no web UI support for previewing large files, and no concept of "tracks" or `.gitattributes`-based auto-detection. You'd be managing everything manually through the CLI or API. - -**Why Git LFS:** LFS is the ecosystem default. GitHub, GitLab, Bitbucket, and Gitea all speak the LFS protocol natively. File locking prevents concurrent binary edits. `.gitattributes` tracks patterns automatically. The web UI shows LFS-tracked files with download links. It's less capable (no encryption, no compression, no chunking), but it's the path of least resistance for teams on hosted Git platforms. - ---- - -### 3. You need to distribute files across dozens of storage backends - -**Use instead: git-annex** - -**Scenario:** You're managing a research dataset that's replicated across 5 university servers (rsync), 2 S3 buckets, a WebDAV share, and a colleague's external hard drive. You need to track which copies exist where, enforce minimum replica counts, and selectively sync subsets. - -**Why not git-cas:** git-cas stores everything in the Git ODB. The only "remote" is wherever you `git push` to. There's no concept of special remotes, location tracking, numcopies enforcement, or selective sync. Your data goes where Git goes, period. - -**Why git-annex:** This is git-annex's entire reason for existing. It supports S3, rsync, WebDAV, Tahoe-LAFS, bittorrent, IPFS, rclone, and custom external remotes. `git annex whereis` tells you which remotes hold each file. `numcopies` ensures a minimum replica count. `git annex get --from=university-server` fetches specific files from specific remotes. No other tool in this space comes close to this level of distributed file management. - ---- - -### 4. You're building ML pipelines with experiment tracking - -**Use instead: DVC** - -**Scenario:** You're training ML models. You want to version your training data, track hyperparameters, compare metrics across experiments, and reproduce any previous run. Your data lives on S3 and your code is on GitHub. - -**Why not git-cas:** git-cas is a content-addressed blob store. It doesn't understand what a "parameter" is, what a "metric" is, or what a "pipeline stage" is. It can store your model weights, but it can't track which hyperparameters produced them, compare accuracy across runs, or rerun a training pipeline. - -**Why DVC:** DVC was purpose-built for this workflow. `dvc.yaml` defines pipeline stages with dependencies and outputs. `dvc exp run` executes experiments with parameter variations. `dvc metrics diff` compares runs. `dvc plots` visualizes training curves. DVCLive integrates with PyTorch and TensorFlow for live logging. The Python API (`dvc.api.open()`) reads versioned data from any DVC remote. It's the MLOps standard. - ---- - -### 5. You need to encrypt files for specific people using their SSH or PGP keys - -**Use instead: Age** - -**Scenario:** You want to encrypt a file so that Alice (SSH key), Bob (age key), and Carol (YubiKey) can all decrypt it. You don't need storage, chunking, or manifests — just "encrypt this file for these three people." - -**Why not git-cas:** git-cas v2.0 uses a single symmetric key. There's no concept of recipients, public-key encryption, or hardware tokens. M11 Locksmith will add multi-recipient via DEK/KEK, but it won't support SSH keys, X25519 identity files, or YubiKey PIV — it's symmetric KEKs only. - -**Why Age:** Age is a pure encryption primitive that does one thing exceptionally well. `age -r ssh-ed25519:AAAA... -r age1... -o secret.enc secret.txt` encrypts for two recipients using their existing keys. The `age-plugin-yubikey` adds hardware token support. Implementations exist in Go, Rust, JavaScript, Java, and Python. It's heading toward post-quantum readiness with X-Wing. If your problem is "encrypt a file for specific people," Age is the answer. - ---- - -### 6. Your repo has >10 GB of binary assets and you need fast clone times - -**Use instead: Git LFS or DVC** - -**Scenario:** Your game repo has 50 GB of textures and audio. New developers need to clone the repo and start working without downloading all 50 GB upfront. - -**Why not git-cas:** git-cas stores blobs in the Git ODB. `git clone` downloads everything. There's no lazy fetching, no sparse checkout for binary assets, no "download only what you need." The vault keeps objects reachable, which means `git gc` won't prune them, which means every clone gets the full history of every binary asset. - -**Why Git LFS / DVC:** Both use pointer files in the Git tree and store actual content externally. `git clone` downloads only the small pointer files. `git lfs pull` or `dvc pull` fetches the actual content on demand, optionally filtered by path or pattern. For very large asset repositories, this deferred-fetch model is essential for developer productivity. - ---- - -## Decision flowchart - -```text -Do you need encrypted binary storage inside Git's ODB? -├── YES → git-cas -│ ├── Need multi-recipient? → Wait for M11 or use Age for the encryption layer -│ ├── Need CDC dedup? → Wait for M10 -│ └── Need >10 GB with lazy clone? → git-cas is the wrong tool. Use LFS + separate encryption -│ -├── NO, I just need images/demos in my README -│ └── Orphan branch (git checkout --orphan assets). Zero dependencies, GitHub renders inline. -│ -├── NO, I need filesystem backups -│ └── Restic -│ -├── NO, I need large files on GitHub with team workflows -│ └── Git LFS -│ -├── NO, I need distributed file replication across many backends -│ └── git-annex -│ -├── NO, I need ML pipeline tracking + data versioning -│ └── DVC -│ -└── NO, I just need to encrypt a file for specific people - └── Age -``` - ---- - -## The honest assessment - -git-cas occupies a specific niche: **Git-native encrypted content-addressed storage for people who want one system, not two.** It's not the best backup tool (Restic is). It's not the best large-file-on-GitHub tool (LFS is). It's not the best distributed file manager (git-annex is). It's not the best ML data versioner (DVC is). It's not the best encryption primitive (Age is). - -What it is: the only tool that lets you `git cas store ./model.bin --slug v3-weights --tree --vault-passphrase "secret"`, commit the tree OID, push to any Git remote, and restore it on any machine with `git cas restore --slug v3-weights --out ./model.bin --vault-passphrase "secret"` — no server, no external storage, no second system. Everything is Git objects, Git refs, Git transport. - -If that's what you want, nothing else does it. If it's not, the right tool probably isn't git-cas. - ---- - -# M13 — Bijou (v3.1.0) ✅ CLOSED - -All tasks completed (13.1–13.6). See [COMPLETED_TASKS.md](./COMPLETED_TASKS.md). - ---- - -## M15 — Prism (code hygiene) ✅ - -Consistency and DRY fixes surfaced by architecture audit. No new features, no API changes. - -### 15.1 — Consistent async `sha256()` across CryptoPort adapters ✅ - -**Problem:** `NodeCryptoAdapter.sha256()` returns `string` (sync), while `BunCryptoAdapter` and `WebCryptoAdapter` return `Promise`. Callers must defensively `await` every call. This is a Liskov Substitution violation — adapters are not interchangeable without the caller knowing which one it got. - -**Fix:** Make `NodeCryptoAdapter.sha256()` return `Promise` (wrap the sync result). All three adapters then have the same async signature. CasService already awaits every call via `_sha256()`, so no call-site changes are needed outside the adapter itself. - -**Files:** -- `src/infrastructure/adapters/NodeCryptoAdapter.js` — add `async` keyword -- `src/ports/CryptoPort.js` — update JSDoc to document `Promise` as the contract -- `src/domain/services/CasService.d.ts` — update `CryptoPort.sha256` type signature - -**Risk:** None. All callers already `await`. Changing sync→async is backward compatible for awaiting code. - -**Tests:** Explicit `expect(adapter.sha256(buf)).toBeInstanceOf(Promise)` test added. - -### 15.2 — Extract `KeyResolver` from CasService ✅ - -**Problem:** `CasService` is a ~1085-line god object. Key resolution logic (~170 lines) is a distinct responsibility: "given a manifest and caller-provided credentials, produce the decryption key." - -**Fix:** Extracted `KeyResolver` class into `src/domain/services/KeyResolver.js`. Receives `CryptoPort` via constructor injection. CasService delegates via `this.keyResolver`. - -**Extracted methods:** -- `validateKeySourceExclusive()` (static) — mutual-exclusion guard -- `wrapDek()` / `unwrapDek()` — DEK envelope operations -- `resolveForDecryption()` — manifest + credentials → decryption key -- `resolveForStore()` — key/passphrase → store-ready key + metadata -- `resolveRecipients()` — multi-recipient DEK generation -- `resolveKeyForRecipients()` — envelope unwrap with fallback iteration -- `#resolvePassphraseForDecryption()` — passphrase → key via manifest KDF -- `#resolveKeyFromPassphrase()` — passphrase + KDF params → derived key - -**Files:** -- New: `src/domain/services/KeyResolver.js` -- Modified: `src/domain/services/CasService.js` — 1085 → 909 lines -- New: `test/unit/domain/services/KeyResolver.test.js` — 24 tests - -**Risk:** None — internal refactor, no public API change. CasService methods remain unchanged. - ---- - -## Backlog (unscheduled) - -Ideas for future milestones. Not committed, not prioritized — just captured. - -### CLI Parity *(recently shipped)* -All library-level configuration is now accessible from the CLI: `--gzip`, `--strategy`, `--chunk-size`, `--concurrency`, `--codec`, `--merkle-threshold`, CDC parameters, `--max-restore-buffer`. Project config file (`.casrc`) provides repository-level defaults. See V9–V12 for remaining CLI gaps. - -### Named Vaults -Multiple vaults instead of one. Refs move from `refs/cas/vault` to `refs/cas/vaults/`. Default vault is `default`. CLI gets `--vault ` flag. - -### Export -- **Export vault to archive** — `git cas vault export --format tar.gz` dumps all entries to a tarball/zip. -- **Export individual entry** — `git cas export --slug photos/vacation --format tar.gz` restores and archives a single entry. -- **Bulk export** — restore multiple slugs into a single archive. - -### Vault Management -- **Move into vault** — `git cas vault add --slug --oid ` to adopt an existing CAS tree into the vault (the API `addToVault()` already supports this; just needs a CLI command). -- **Vault status** — `git cas vault status` shows metadata, `encryptionCount`, entry count, nonce health. See V9. -- **Purge from CAS** — remove an entry from the vault and run `git gc` to reclaim storage. See V10. - -### Publish / Mount -- **Publish to working tree** — `git cas publish --slug assets/hero --to docs/hero.gif` reconstitutes a vault entry into the repo's working tree so it's servable by GitHub (markdown images, Pages, etc.). -- **Publish to branch** — `git cas publish --branch gh-assets` materializes all vault entries onto a dedicated branch. Keeps the main branch clean while making assets accessible via GitHub raw URLs. -- **Auto-publish hook** — pre-commit or CI step that keeps published assets in sync with vault state. - -### Repo Intelligence -- **Duplicate detection on store** — warn if a file being stored already exists as a tracked git blob (same content hash). "This file is already tracked by git — are you sure you want to store it in CAS too?" -- **Repo scan / dedup advisor** — `git cas scan` walks the git object database and recommends files that could benefit from CAS (large blobs, binary files, duplicated content across branches). Reports dedup opportunities and potential storage savings. - ---- - -# Ideas & Visions - -New feature concepts with fully fleshed out visions and mini battle plans. Not committed to any milestone — captured here for future consideration and discussion. - ---- - -## Vision 1: Snapshot Trees — Directory-Level Store - -**The Pitch** - -Today, git-cas stores one file at a time. Storing a build output directory means N separate `storeFile()` calls, N separate vault entries, and the caller manually tracking which slugs belong together. There's no concept of "this set of files is one atomic artifact." - -Snapshot trees change that. `git cas store-tree ./dist --slug release/v4.0.0` stores an entire directory as a single CAS tree — one root manifest that references child manifests per file, one vault entry, one tree OID. Restore reconstitutes the full directory structure. This unlocks "store my build output" as a single atomic operation. - -**Why It Matters** - -- **CI/CD artifacts**: `npm run build && git cas store-tree ./dist --slug build/$CI_COMMIT_SHA --tree` — one command, one OID, committed in the release tag. -- **Dataset versioning**: Store a directory of CSV/Parquet files as a single versioned snapshot, restore any version atomically. -- **Config bundles**: Store an entire config directory (TLS certs, env files, deploy scripts) as one encrypted vault entry. -- **Binary releases**: Store a multi-file release (binary + README + license + checksums) as one restorable unit. - -**Manifest Design** - -```json -{ - "version": 3, - "type": "tree", - "slug": "release/v4.0.0", - "entries": [ - { "path": "index.js", "manifestOid": "abc123...", "size": 45200 }, - { "path": "lib/utils.js", "manifestOid": "def456...", "size": 12800 }, - { "path": "assets/logo.png", "manifestOid": "789abc...", "size": 204800 } - ], - "totalSize": 262800, - "totalChunks": 12, - "encryption": { ... }, - "compression": { ... } -} -``` - -Each `entries[].manifestOid` points to a standard file-level manifest blob (v1/v2). The root tree manifest is the index; child manifests are the per-file metadata. Encryption and compression applied per-file, configured at the tree level. - -**Mini Battle Plan** - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. Schema | Add `TreeManifestSchema` with `type: 'tree'`, `entries[]` array. Backward compat: existing manifests have no `type` field (treated as `type: 'file'`). | ~40 | ~2h | -| 2. CasService | `storeTree({ source: string, slug, encryptionKey?, compression? })` — walks directory recursively, stores each file via existing `store()`, collects child manifests, builds root tree manifest. Parallel file stores via semaphore. | ~120 | ~6h | -| 3. CasService | `restoreTree({ manifest, outputDir, encryptionKey? })` — reads root tree manifest, restores each child file to `outputDir/entry.path`. Creates intermediate directories. | ~80 | ~4h | -| 4. Facade | Wire `storeDirectory()` and `restoreDirectory()` through `ContentAddressableStore`. | ~30 | ~1h | -| 5. CLI | `git cas store-tree --slug ` and `git cas restore --slug --out ` (auto-detect tree vs file manifest). | ~40 | ~2h | -| 6. Tests | Round-trip with nested dirs, empty dirs, symlinks (skip or follow?), encrypted+compressed trees, Merkle child manifests. | ~100 | ~4h | -| **Total** | | **~410** | **~19h** | - -**Open Questions** -- Symlinks: follow, skip, or store as metadata? -- Empty directories: include in manifest or skip? -- File permissions: record and restore, or ignore? -- Maximum depth limit to prevent unbounded recursion? - ---- - -## Vision 2: Portable Bundles — Air-Gap Transfer - -**The Pitch** +| Version | Milestone | Codename | Theme | Status | +|---------|-----------|----------|-------|--------| +| v3.1.0 | M13 | Bijou | TUI dashboard and animated progress | ✅ Shipped | +| v4.0.0 | M14 | Conduit | Streaming restore, observability, parallel chunk I/O | ✅ Shipped | +| v4.0.1 | M8 + M9 | Spit Shine + Cockpit | Review hardening, `verify`, `--json`, CLI polish | ✅ Shipped | +| v5.0.0 | M10 | Hydra | Content-defined chunking | ✅ Shipped | +| v5.1.0 | M11 | Locksmith | Envelope encryption and recipient management | ✅ Shipped | +| v5.2.0 | M12 | Carousel | Key rotation without re-encrypting data | ✅ Shipped | +| v5.3.0 | M16 | Capstone | Audit remediation and security hardening | ✅ Shipped | +| v5.3.1 | — | Maintenance | Repeated-chunk tree integrity fix | ✅ Shipped | -`git cas bundle --slug my-asset --out asset.casb` creates a self-contained bundle file that includes the manifest, all chunk blobs, and enough metadata to import into any git-cas-enabled repo. `git cas import --bundle asset.casb` reconstitutes it. Like `git bundle` but for CAS assets. +Older history remains in [CHANGELOG.md](./CHANGELOG.md). -This enables offline transfer between air-gapped systems without needing `git push/pull` or shared remotes. Ship a USB stick, email an encrypted bundle, or distribute via any file transfer mechanism. +## Planned Release Sequence -**Bundle Format** +| Version | Milestone | Codename | Theme | Status | +|---------|-----------|----------|-------|--------| +| v5.3.2 | M17 | Ledger | Planning and ops reset | 📝 Planned | +| v5.4.0 | M18 | Relay | LLM-native CLI foundation | 📝 Planned | +| v5.5.0 | M19 | Nouveau | Bijou v3 human UX refresh | 📝 Planned | +| v5.6.0 | M20 | Sentinel | Vault health and safety | 📝 Planned | +| v5.7.0 | M21 | Atelier | Vault ergonomics and publishing | 📝 Planned | +| v5.8.0 | M22 | Cartographer | Repo intelligence and change analysis | 📝 Planned | +| v5.9.0 | M23 | Courier | Artifact sets and transfer | 📝 Planned | +| v5.10.0 | M24 | Spectrum | Storage and observability extensibility | 📝 Planned | +| v5.11.0 | M25 | Bastion | Enterprise key management research | 📝 Planned | + +## Dependency Sequence ```text -┌─────────────────────────────┐ -│ Magic: "CASB\x01" (5B) │ ← Version 1 bundle -│ Header length (4B) │ -│ Header (JSON): │ -│ { slug, filename, size, │ -│ chunkCount, encrypted, │ -│ compressed, codec } │ -│ Manifest blob (var) │ -│ Chunk 0 length (4B) │ -│ Chunk 0 data (var) │ -│ Chunk 1 length (4B) │ -│ Chunk 1 data (var) │ -│ ... │ -│ SHA-256 checksum (32B) │ ← Over everything above -└─────────────────────────────┘ -``` - -Simple, streamable, no external dependencies. The checksum at the end covers the entire bundle — tamper detection without needing encryption. - -**Mini Battle Plan** - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. Format spec | Define bundle wire format, version byte, header schema. Document in `docs/BUNDLE-FORMAT.md`. | ~0 prod, ~40 docs | ~1h | -| 2. Bundle writer | `CasService.createBundle({ manifest, output: WritableStream })` — streams manifest + chunks into bundle format. Calculates trailing checksum. | ~80 | ~4h | -| 3. Bundle reader | `CasService.importBundle({ input: ReadableStream })` — parses header, validates checksum, writes blobs to Git ODB, returns manifest. | ~100 | ~5h | -| 4. Facade + CLI | `git cas bundle --slug --out ` and `git cas import --bundle [--vault]`. | ~40 | ~2h | -| 5. Tests | Round-trip, corrupted bundle (bad checksum), encrypted bundles, Merkle manifests, partial read (truncated file). | ~80 | ~3h | -| **Total** | | **~340** | **~15h** | - -**Why Not Just `git bundle`?** - -`git bundle` exports entire ref histories. It requires the recipient to have a compatible Git repo structure. CAS bundles export a single asset with just the manifest and blobs — no ref history, no commit chain, no pack negotiation. They're smaller, simpler, and purpose-built for asset transfer. - ---- - -## Vision 3: Manifest Diff Engine - -**The Pitch** - -`git cas diff --from photos/v1 --to photos/v2` compares two manifests and shows which chunks changed, were added, or removed. With CDC (M10), this becomes extremely powerful — you can see exactly which byte ranges of a binary file changed between versions. - -**API Design** - -```js -const diff = await cas.diffManifests({ oldManifest, newManifest }); -// Returns: -// { -// unchanged: [{ index, digest, size }], -// added: [{ index, digest, size }], -// removed: [{ index, digest, size }], -// modified: [{ oldIndex, newIndex, oldDigest, newDigest }], -// summary: { -// unchangedBytes: 1048576, -// addedBytes: 262144, -// removedBytes: 0, -// reuseRatio: 0.8, // 80% of chunks reused -// } -// } -``` - -The diff is purely metadata-based — no blob reads required. Compare chunk digests between manifests. With fixed chunking, any insertion shifts all downstream chunks (low reuse). With CDC, insertions affect 1-2 chunks (high reuse). The `reuseRatio` metric quantifies dedup efficiency. - -**TUI Integration** - -``` -git cas diff --from photos/v1 --to photos/v2 --heatmap - - v1: ████████████████████████████████████████ - v2: ████████░░░░████████████████████████████ - ^^^^ - 2 chunks changed (bytes 524288–786432) - 38/40 chunks reused (95.0%) -``` - -**Mini Battle Plan** - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. Diff engine | `CasService.diffManifests({ oldManifest, newManifest })` — digest-set comparison, handles reordering. | ~60 | ~3h | -| 2. CLI command | `git cas diff --from --to ` with human-readable summary. | ~30 | ~1h | -| 3. Heatmap view | Side-by-side chunk heatmap showing unchanged (green), changed (red), added (yellow) blocks. Reuses bijou gradient components from Task 13.5. | ~40 | ~2h | -| 4. Tests | Identical manifests (0 diff), completely different, single-chunk change, CDC vs fixed dedup comparison. | ~50 | ~2h | -| **Total** | | **~180** | **~8h** | - -**Synergy with M10 Hydra**: Diff becomes dramatically more useful with CDC. Fixed chunking: insert 1 byte → 100% of downstream chunks changed. CDC: insert 1 byte → 1-2 chunks changed. The diff engine quantifies this, making CDC's value proposition concrete and measurable. - ---- - -## Vision 4: CompressionPort — zstd, brotli, lz4 - -**The Pitch** - -Currently, compression is hardcoded to gzip. The `CompressionPort` abstraction mirrors the existing `CryptoPort` and `CodecPort` patterns — a port with pluggable adapters. The manifest already records `compression.algorithm`, so backward compat is built in. - -**Why This Matters** - -| Algorithm | Ratio (typical) | Compress speed | Decompress speed | Best for | -|-----------|-----------------|----------------|------------------|----------| -| gzip | Good | Slow (~50 MB/s) | Moderate (~300 MB/s) | Current default | -| **zstd** | **Excellent** | **Fast (~500 MB/s)** | **Very fast (~1.5 GB/s)** | **General purpose, best all-rounder** | -| brotli | Excellent | Very slow (~10 MB/s) | Fast (~500 MB/s) | Pre-compressed web assets | -| lz4 | Moderate | Ultra-fast (~2 GB/s) | Ultra-fast (~4 GB/s) | Speed-critical, low-latency | - -Zstd alone would give 5-10x faster compression with equal or better ratio. For a tool that compresses before encrypting, compression speed directly impacts store throughput. - -**Mini Battle Plan** - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. Port definition | `src/ports/CompressionPort.js` — `compress(source: AsyncIterable): AsyncIterable` and `decompress(buffer: Buffer): Promise`. Property: `algorithm: string`. | ~20 | ~1h | -| 2. GzipAdapter | Wrap existing `createGzip()` / `gunzipAsync()` logic into adapter. Remove inline gzip from CasService. | ~30 | ~1h | -| 3. ZstdAdapter | Use `@napi-rs/zstd` (native binding, 0-dep) or `fzstd` (pure JS fallback). Streaming compress via transform. | ~40 | ~2h | -| 4. CasService refactor | Replace inline compression with `this.compression.compress(source)` and `this.compression.decompress(buffer)`. Facade accepts `compression: { algorithm: 'gzip' \| 'zstd' }` and selects adapter. | ~30 | ~2h | -| 5. Tests + benchmarks | Round-trip with each algorithm. Benchmark: 10 MB file, gzip vs zstd compress speed and ratio. | ~60 | ~2h | -| **Total** | | **~180** | **~8h** | - -**Backward Compatibility**: Old manifests with `compression.algorithm: 'gzip'` still work — the facade selects the gzip adapter. New manifests can specify `'zstd'`. Restoring always reads the algorithm from the manifest, so mixed-algorithm vaults work seamlessly. - ---- - -## Vision 5: Watch Mode — Continuous Sync - -**The Pitch** - -`git cas watch ./data --slug live-data --interval 5s` monitors a file or directory for changes and incrementally re-stores modified content. Combined with CDC (M10), only changed chunks get written. The vault entry updates atomically on each sync cycle. - -**Use Cases** - -- **Development hot-reload**: Watch a model weights file during training; each checkpoint auto-stored with a versioned slug (`live-data@1`, `live-data@2`, ...). -- **Config sync**: Watch a config directory; changes automatically vaulted. -- **Continuous backup**: Low-overhead continuous protection for critical files. - -**Mini Battle Plan** - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. File watcher | Use `fs.watch()` (Node) / `Bun.file().watch()` with debounce (default 1s). Detect create/modify/delete. | ~50 | ~2h | -| 2. Incremental store | On change: re-store the file, diff manifests (Vision 3), skip if unchanged (identical digest). Update vault entry with `--force`. | ~60 | ~3h | -| 3. CLI command | `git cas watch --slug [--interval ] [--key-file ]`. Ctrl-C to stop. | ~30 | ~1h | -| 4. Progress | Live status line showing: last sync time, files watched, total syncs, bytes stored. | ~20 | ~1h | -| 5. Tests | Mock fs.watch, verify debounce, verify vault updates, verify no-op on unchanged files. | ~60 | ~3h | -| **Total** | | **~220** | **~10h** | - -**Synergy with CDC (M10)**: Without CDC, every modification re-stores every chunk downstream of the edit point. With CDC, only 1-2 chunks change per modification. Watch mode + CDC together give efficient continuous incremental storage. - ---- - -## Vision 6: Interactive Passphrase Prompt ✅ DONE - -**Status:** Implemented by Task 16.11. See `bin/ui/passphrase-prompt.js`. - -Passphrase resolution priority: `--vault-passphrase-file` → `--vault-passphrase` → `GIT_CAS_PASSPHRASE` → interactive TTY prompt. Confirmation prompt on first use (vault init). File permission warnings on group/world-readable passphrase files. CRLF normalization for Windows compatibility. - ---- - -## Vision 9: Vault Status Command - -**The Pitch** - -`git cas vault status` — a single command to show vault health: entry count, encryption state, `encryptionCount` (nonce usage), KDF parameters, and nonce health assessment. The `encryptionCount` and `decryption_failed` metrics from M16 are already tracked in vault metadata but have no CLI surface. - -```shell -$ git cas vault status -Entries: 42 -Encryption: aes-256-gcm (pbkdf2, 600000 iterations) -Encryption count: 1,247 / 2,147,483,648 (0.00%) -Nonce health: ✅ Safe (rotate key before 2^31) -``` - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. Command + metadata display | Read vault metadata, format output (text + JSON) | ~40 | ~1h | -| 2. Nonce health assessment | Compare `encryptionCount` against thresholds, emit warning levels | ~15 | ~0.5h | -| 3. Tests | Mock vault state, verify output format | ~20 | ~0.5h | -| **Total** | | **~60** | **~2h** | - ---- - -## Vision 10: GC Command - -**The Pitch** - -`git cas gc` — identify unreferenced chunks across vault entries and optionally trigger `git gc`. Wraps `collectReferencedChunks()` with a user-facing report. - -```shell -$ git cas gc --dry-run -Referenced chunks: 1,247 -Unreferenced blobs: 23 (estimated 4.2 MiB) -Run without --dry-run to trigger git gc. -``` - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. Chunk analysis | Call `collectReferencedChunks` for all vault entries, compare against all CAS blobs | ~40 | ~2h | -| 2. `git gc` integration | Optionally invoke `git gc` after analysis. `--dry-run` flag for preview. | ~20 | ~1h | -| 3. Tests + safety | Confirm dry-run default, test output format | ~20 | ~1h | -| **Total** | | **~80** | **~4h** | - ---- - -## Vision 11: KDF Parameter Tuning via `.casrc` - -**The Pitch** - -Allow `.casrc` to specify KDF parameters for vault init and rotation: `kdf.algorithm`, `kdf.iterations` (PBKDF2), `kdf.cost`/`kdf.blockSize`/`kdf.parallelization` (scrypt). Reject insecure values below OWASP minimums (100,000 iterations for PBKDF2-SHA-512, N=8192 for scrypt). - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. Config schema + validation | Add `kdf` key to `.casrc` schema, validate against OWASP minimums | ~25 | ~1h | -| 2. Wire into vault init/rotate | `loadConfig` merges KDF params into `kdfOptions` | ~10 | ~0.5h | -| 3. Tests | Reject weak params, merge precedence | ~15 | ~0.5h | -| **Total** | | **~40** | **~2h** | - ---- - -## Vision 12: File-Level Passphrase CLI - -**The Pitch** - -`--passphrase` flag for standalone encrypted store without requiring vault encryption. The library already supports `passphrase` + `kdfOptions` on `store()`/`storeFile()` — this just wires it through the CLI. - -```shell -# Store with a one-off passphrase (not vault-level) -git cas store ./secrets.bin --slug one-off --passphrase "my secret" - -# Restore with the same passphrase -git cas restore --slug one-off --out ./secrets.bin --passphrase "my secret" +M16 Capstone + v5.3.1 maintenance ✅ + | + M17 Ledger + | + M18 Relay + | + M19 Nouveau + | + M20 Sentinel + | + M21 Atelier + | + M22 Cartographer + | + M23 Courier + | + M24 Spectrum + | + M25 Bastion ``` -Mutually exclusive with `--key-file`, `--recipient`, and `--vault-passphrase`. Supports the same resolution chain as vault passphrases: `--passphrase-file`, `--passphrase`, env var `GIT_CAS_FILE_PASSPHRASE`, TTY prompt. - -| Phase | Work | ~LoC | ~Hours | -|-------|------|------|--------| -| 1. CLI flag + store wiring | Add `--passphrase` to store/restore, wire into `storeFile`/`restoreFile` with `passphrase` + `kdfOptions` | ~20 | ~0.5h | -| 2. Mutual exclusion validation | Reject conflicting encryption flags | ~5 | ~0.25h | -| 3. Tests | Store/restore round-trip, conflict validation | ~15 | ~0.5h | -| **Total** | | **~30** | **~1h** | +This sequence is intentionally linear. It forces the docs/ops reset first, then the machine +interface split, then the human TUI refresh, and only then the broader feature expansion. ---- +## Open Milestones -# Concerns & Mitigations +### M17 — Ledger (`v5.3.2`) -Architectural and security concerns identified during code review, with proposed mitigations and defensive tests for each. +**Theme:** planning and operational reset after Capstone. ---- +Deliverables: -## Concern 1: Memory Amplification on Encrypted/Compressed Restore ✅ MITIGATED +- Close M16 in docs and reconcile [ROADMAP.md](./ROADMAP.md), [STATUS.md](./STATUS.md), and the shipped version history. +- Add `CODEOWNERS` or equivalent review-assignment automation. +- Document Git tree filename ordering semantics in test conventions to prevent future false positives. +- Define a release-prep workflow for `CHANGELOG` updates and version bump timing. +- Automate test-count injection into release notes or changelog prep. +- Add property-based fuzz coverage for envelope-encryption round-trips. -**Status:** Task 16.2 implemented `maxRestoreBufferSize` guard (default 512 MiB). Post-decompression guard added. CLI exposes `--max-restore-buffer`. +### M18 — Relay (`v5.4.0`) -**The Problem** +**Theme:** first-class LLM-native CLI. -`_restoreBuffered()` concatenates ALL chunk blobs into a single buffer, decrypts, then decompresses. Despite `restoreStream()` exposing an `AsyncIterable` API (implying streaming), encrypted or compressed files buffer the entire plaintext in memory. A 10 GB encrypted file attempts a 10 GB allocation — and then potentially a second 10 GB buffer for decompression. +Deliverables: -The JSDoc note added in the M14 review documents this, but there's no runtime guard. A user calling `restoreStream()` expecting constant memory will OOM silently on large encrypted files. +- Introduce `git cas agent` as a separate machine-facing namespace. +- Add a dedicated machine command runner instead of extending the current human `runAction()` path. +- Define and implement the JSONL envelope contract: + `protocol`, `command`, `type`, `seq`, `ts`, `data`. +- Implement reserved record types: + `start`, `progress`, `warning`, `needs-input`, `result`, `error`, `end`. +- Enforce non-interactive behavior for secrets and missing inputs. +- Support flags plus `--request -` / `--request @file.json`. +- Deliver parity for: + `agent store`, `agent tree`, `agent inspect`, `agent restore`, `agent verify`, + `agent vault list`, `agent vault info`, `agent vault history`. +- Publish contract docs with exact exit-code behavior. -**Root Cause**: AES-256-GCM requires the entire ciphertext for authentication tag verification before any plaintext is released. You can't verify-then-stream with GCM — it's authenticate-everything-or-nothing. This is a fundamental limitation of the cipher mode, not a bug. +Acceptance: -**Mitigation Strategy** +- JSONL contract tests must verify record order, record shapes, `stdout` purity, `stderr` silence after protocol start, and exit codes on Node, Bun, and Deno. -| # | Mitigation | Effort | Impact | -|---|-----------|--------|--------| -| M1 | **Memory guard**: Add `maxRestoreBufferSize` option (default 512 MB). `_restoreBuffered()` checks `manifest.size` against limit before allocating. Throws `CasError('RESTORE_TOO_LARGE')` with actionable message suggesting chunked storage without encryption, or increasing the limit. | ~20 LoC | Prevents surprise OOM | -| M2 | **Per-chunk encryption** (long-term): Encrypt each chunk independently with a derived per-chunk nonce (`baseNonce + chunkIndex`). Each chunk gets its own GCM tag. Restore can verify and decrypt per-chunk in O(chunkSize) memory. **Breaking change** — new manifest encryption format. | ~200 LoC | True streaming encrypted restore | -| M3 | **Documentation**: Add a "Memory Model" section to README explaining which code paths buffer and which stream. | ~0 LoC | Sets expectations | +### M19 — Nouveau (`v5.5.0`) -**Recommended**: M1 (immediate safety net) + M3 (documentation). M2 is a future milestone — it changes the encryption format and requires careful security analysis (per-chunk nonces must not collide, chunk reordering attacks need mitigation via a MAC over the chunk sequence). - -**Defensive Tests** - -```js -describe('Concern 1: Memory guard on encrypted restore', () => { - it('throws RESTORE_TOO_LARGE when manifest.size exceeds maxRestoreBufferSize', ...); - it('succeeds when manifest.size is within maxRestoreBufferSize', ...); - it('does not apply guard to unencrypted uncompressed restoreStream', ...); - it('includes actionable hint in RESTORE_TOO_LARGE error message', ...); -}); -``` - -**New Error Code**: `RESTORE_TOO_LARGE` — "File too large for buffered restore. The encrypted/compressed restore path buffers the entire file in memory. Set `maxRestoreBufferSize` to increase the limit, or store without encryption for streaming restore." - ---- - -## Concern 2: Orphaned Blob Accumulation After STREAM_ERROR ✅ MITIGATED - -**Status:** Task 16.10 implemented. `STREAM_ERROR` meta includes `orphanedBlobs` array. Observability metric emitted. - -**The Problem** - -When `_chunkAndStore()` throws `STREAM_ERROR`, chunks already written to Git via `persistence.writeBlob()` are orphaned — they exist in the Git ODB but no tree or ref references them. `git gc` will eventually reclaim them (default grace period: 2 weeks), but: - -1. No tracking of which blobs were orphaned — there's no cleanup manifest or error log. -2. In high-failure environments (unreliable sources, network streams), orphaned blobs accumulate silently. -3. `git count-objects` shows growing "loose objects" with no explanation. - -The `await Promise.allSettled(pending)` fix from C1 ensures in-flight writes complete (no floating promises), but their results are discarded — successful writes still create orphaned blobs. - -**Mitigation Strategy** - -| # | Mitigation | Effort | Impact | -|---|-----------|--------|--------| -| M1 | **Report orphaned blobs in error metadata**: After `Promise.allSettled(pending)`, collect the blob OIDs from fulfilled results and include them in the `STREAM_ERROR` meta: `{ chunksDispatched, orphanedBlobs: ['abc...', 'def...'], originalError }`. Callers can log or clean up. | ~15 LoC | Visibility | -| M2 | **Observability metric**: Emit `observability.metric('error', { action: 'orphaned_blobs', count: N, blobs: [...] })` so monitoring systems can track accumulation. | ~5 LoC | Monitoring | -| M3 | **CLI warning**: When `git cas store` fails with STREAM_ERROR, print: `"Warning: N chunk blobs were written before the error. They will be reclaimed by 'git gc'."` | ~5 LoC | User awareness | - -**Recommended**: M1 + M2 (cheap, high-value visibility). M3 for CLI polish. - -**Defensive Tests** - -```js -describe('Concern 2: Orphaned blob tracking on STREAM_ERROR', () => { - it('includes orphanedBlobs array in STREAM_ERROR meta', ...); - it('orphanedBlobs contains blob OIDs from successful writes before failure', ...); - it('orphanedBlobs is empty when stream fails before any writes', ...); - it('emits orphaned_blobs metric via observability', ...); -}); -``` +**Theme:** Bijou v3 refresh for the human-facing experience. ---- +Deliverables: -## Concern 3: No Upper Bound on Chunk Size ✅ MITIGATED +- Upgrade `@flyingrobots/bijou`, `@flyingrobots/bijou-node`, and `@flyingrobots/bijou-tui` to `3.0.0`. +- Add `@flyingrobots/bijou-tui-app` for the refreshed shell. +- Move inspector/dashboard rendering onto the v3 `ViewOutput` contract. +- Split the current inspector into sub-apps for list, detail, history, and health panes. +- Add BCSS-driven responsive styling and layout presets. +- Add motion for focus shifts, pane changes, and shell transitions where it improves legibility. +- Add session restore for the human TUI layout. +- Replace the current low-fidelity heatmap/detail composition with a higher-fidelity surface-native view. -**Status:** Task 16.6 implemented. 100 MiB hard cap on CasService, FixedChunker, CdcChunker. Warning at 10 MiB. +Acceptance: -**The Problem** +- Existing human CLI behavior stays stable outside the refreshed TUI. +- PTY smoke coverage must exercise inspect/dashboard navigation, filtering, resize, pane composition, and non-TTY fallback. -`CasService` enforces a minimum chunk size (`chunkSize < 1024` throws), but there's no maximum. A user can configure `chunkSize: 4 * 1024 * 1024 * 1024` (4 GB) — and `git hash-object -w` will attempt to read the entire 4 GB chunk into memory as a single buffer. The `_storeChunk()` method passes the chunk buffer to `persistence.writeBlob()`, which shells out to `git hash-object` via stdin — but the buffer itself is already in Node.js memory. - -Additionally, Git repositories have practical performance limits on individual blob sizes. While there's no hard cap, blobs >100 MB cause significant performance degradation in pack files, and >1 GB blobs can cause `git push` failures on many hosting platforms (GitHub's limit is 100 MB per blob via the API). - -**Mitigation Strategy** - -| # | Mitigation | Effort | Impact | -|---|-----------|--------|--------| -| M1 | **Enforce maximum chunk size**: Add `if (chunkSize > 100 * 1024 * 1024) throw new Error('Chunk size must not exceed 100 MiB')` in the constructor. 100 MiB is generous (default is 256 KiB) while staying within Git hosting limits. | ~3 LoC | Prevents footgun | -| M2 | **Warn above 10 MiB**: Emit `observability.log('warn', 'Large chunk size may impact Git performance', { chunkSize })` when chunkSize > 10 MiB. | ~3 LoC | Soft guidance | - -**Recommended**: M1 (hard cap) + M2 (soft warning). The maximum can be made configurable via an `allowLargeChunks: true` escape hatch for advanced users. - -**Defensive Tests** - -```js -describe('Concern 3: Chunk size upper bound', () => { - it('throws when chunkSize exceeds 100 MiB', ...); - it('accepts chunkSize of exactly 100 MiB', ...); - it('accepts default chunkSize (256 KiB)', ...); - it('accepts minimum chunkSize (1024 bytes)', ...); - it('logs warning when chunkSize exceeds 10 MiB', ...); -}); -``` - ---- - -## Concern 4: Web Crypto Adapter Silent Memory Buffering ✅ MITIGATED - -**Status:** Task 16.3 implemented. `ENCRYPTION_BUFFER_EXCEEDED` thrown when accumulated bytes exceed `maxEncryptionBufferSize` (default 512 MiB). - -**The Problem** - -`WebCryptoAdapter.createEncryptionStream()` returns an `encrypt()` async generator that appears to stream, but internally accumulates all chunks into a single buffer before calling `crypto.subtle.encrypt()` (which is one-shot for GCM). The Deno runtime uses this adapter. A user on Deno calling `store()` with a 5 GB source will OOM without any indication that streaming is not actually happening. - -The NodeCryptoAdapter and BunCryptoAdapter use `node:crypto` Cipher streams which truly stream — so this is a Deno-specific behavioral difference with no warning. - -**Mitigation Strategy** - -| # | Mitigation | Effort | Impact | -|---|-----------|--------|--------| -| M1 | **Size tracking in encrypt generator**: Track accumulated bytes in `encrypt()`. When total exceeds a configurable limit (default 512 MB), throw `CasError('ENCRYPTION_BUFFER_EXCEEDED')` with message: `"Web Crypto API requires buffering the entire file for GCM encryption. File exceeds buffer limit. Use Node.js or Bun for large encrypted files, or store without encryption."` | ~15 LoC | Prevents silent OOM | -| M2 | **Runtime capability flag**: Add `CryptoPort.capabilities` property: `{ streamingEncryption: boolean }`. WebCryptoAdapter returns `false`. CasService can check this and warn or error when storing large encrypted files on non-streaming runtimes. | ~20 LoC | Architectural awareness | -| M3 | **Adapter-level documentation**: JSDoc on WebCryptoAdapter noting the buffering limitation. | ~5 LoC | Developer awareness | - -**Recommended**: M1 (safety net) + M3 (documentation). M2 is a clean long-term solution. - -**Defensive Tests** - -```js -describe('Concern 4: Web Crypto buffering guard', () => { - it('throws ENCRYPTION_BUFFER_EXCEEDED when accumulated bytes exceed limit', ...); - it('succeeds for files within buffer limit', ...); - it('NodeCryptoAdapter does NOT throw for large files (true streaming)', ...); - it('WebCryptoAdapter.capabilities.streamingEncryption is false', ...); -}); -``` - -**New Error Code**: `ENCRYPTION_BUFFER_EXCEEDED` — "File exceeds encryption buffer limit on this runtime. Web Crypto API (Deno) buffers the entire file for AES-GCM. Use Node.js or Bun for large encrypted files." - ---- - -## Concern 5: Passphrase Exposure in Shell History and Process Listings ✅ MITIGATED - -**Status:** Task 16.11 implemented. `--vault-passphrase-file`, interactive TTY prompt, stdin pipe. See Vision 6. - -**The Problem** - -The `--vault-passphrase ` CLI flag puts the passphrase in: -1. **Shell history**: `~/.bash_history`, `~/.zsh_history` — survives terminal close, searchable. -2. **Process listing**: `ps aux` shows full command line including the passphrase to all users on the system. -3. **CI logs**: If used in a CI pipeline without masking, the passphrase appears in build logs. - -The `GIT_CAS_PASSPHRASE` env var is better (not in shell history) but still visible in `/proc//environ` on Linux and in process listings on some systems. - -**Mitigation Strategy** - -| # | Mitigation | Effort | Impact | -|---|-----------|--------|--------| -| M1 | **Interactive prompt**: See Vision 6 above. `--vault-passphrase` without a value triggers TTY prompt with echo disabled. Confirmation on first use. | ~90 LoC | Eliminates history exposure | -| M2 | **File-based passphrase**: `--vault-passphrase-file ` reads the passphrase from a file (like `docker secret`, `kubectl --token-file`). File can be tmpfs-backed, permissions-restricted, or injected by a secrets manager. | ~15 LoC | CI-friendly, no process exposure | -| M3 | **Stdin passphrase**: `echo "secret" \| git cas store --vault-passphrase -` reads from stdin. Useful in pipes. | ~10 LoC | Scriptable | -| M4 | **Documentation warning**: Add security note in README and `--help` output: "Avoid passing passphrases on the command line. Use `GIT_CAS_PASSPHRASE` env var, `--vault-passphrase-file`, or omit the value for interactive prompt." | ~0 LoC | Awareness | - -**Recommended**: M1 + M2 + M4. Interactive prompt for humans, file-based for CI, documentation for everyone. - -**Defensive Tests** - -```js -describe('Concern 5: Passphrase input security', () => { - it('reads passphrase from file when --vault-passphrase-file is used', ...); - it('prompts interactively when --vault-passphrase is passed without value in TTY', ...); - it('falls back to GIT_CAS_PASSPHRASE env var in non-TTY', ...); - it('errors when no passphrase source is available in non-TTY mode', ...); - it('confirmation prompt rejects mismatched passphrases', ...); -}); -``` - ---- - -## Concern 6: No KDF Brute-Force Rate Limiting ✅ MITIGATED - -**Status:** Task 16.12 implemented. `decryption_failed` observability metric. CLI 1s delay on `INTEGRITY_ERROR`. - -**The Problem** - -`deriveKey()` and the restore path's `_resolveKeyFromPassphrase()` have no rate limiting, attempt counting, or lockout mechanism. An attacker with access to the API or CLI can brute-force passphrases at full CPU speed: - -- PBKDF2 (100k iterations, SHA-512): ~100-500 attempts/sec on modern hardware. -- scrypt (N=16384, r=8, p=1): ~10-50 attempts/sec. - -For a strong passphrase (>80 bits of entropy), this is fine — but many users choose weak passphrases. There's no warning, no audit trail, and no way to detect an ongoing brute-force attack. - -**Mitigation Strategy** - -| # | Mitigation | Effort | Impact | -|---|-----------|--------|--------| -| M1 | **Observability metric on failed decryption**: Emit `observability.metric('error', { action: 'decryption_failed', slug, attempt })` on every `INTEGRITY_ERROR` during restore. Monitoring systems can alert on anomalous failure rates. | ~5 LoC | Detection | -| M2 | **CLI rate limit**: In the CLI layer (not the library), add a 1-second delay after each failed passphrase attempt. Prevents rapid brute-force via the terminal without affecting the programmatic API. | ~5 LoC | CLI hardening | -| M3 | **Stronger KDF defaults**: Increase PBKDF2 default iterations from 100k to 600k (OWASP 2023 recommendation for SHA-512). Increase scrypt default cost from 16384 to 65536. Document the change as a security improvement. **Note**: this affects store/restore performance — KDF runs once per operation, so the latency increase (100ms → 600ms for PBKDF2) is acceptable for interactive use but may impact batch workflows. | ~5 LoC | Resistance | -| M4 | **Documentation**: Add KDF parameter guidance to SECURITY.md — recommended iterations/cost for different threat models (personal use, team, high-security). | ~0 LoC | Guidance | - -**Recommended**: M1 (detection) + M2 (CLI hardening) + M4 (guidance). M3 is a judgment call — the performance tradeoff is worth discussing. - -**Defensive Tests** - -```js -describe('Concern 6: KDF brute-force awareness', () => { - it('emits decryption_failed metric on wrong passphrase', ...); - it('emits metric with slug context for audit trail', ...); - it('CLI applies delay after failed passphrase attempt', ...); - it('library API does NOT rate-limit (callers manage their own policy)', ...); -}); -``` - ---- - -## Concern 7: GCM Nonce Collision Risk at Scale ✅ MITIGATED - -**Status:** Task 16.13 implemented. `SECURITY.md` documents GCM bound. Vault tracks `encryptionCount`. Warning at 2^31. - -**The Problem** - -AES-256-GCM uses a 96-bit (12-byte) nonce, generated randomly per `encryptBuffer()` / `createEncryptionStream()` call. The birthday bound for 96-bit nonces is ~2^48 — after ~281 trillion encryptions with the same key, nonce collision probability exceeds 50%. In practice, NIST recommends limiting to 2^32 (~4.3 billion) invocations per key for a negligible collision probability. - -For a single user storing files with one key, this is not a practical concern — you'd need to store 4 billion files. But: -1. There's no explicit tracking or warning of nonce count per key. -2. The nonce is pure random, not a counter — so there's no guarantee of uniqueness even at low counts (just overwhelming probability). -3. A nonce collision with GCM is catastrophic — it reveals the XOR of two plaintexts and allows auth tag forgery. - -**Mitigation Strategy** - -| # | Mitigation | Effort | Impact | -|---|-----------|--------|--------| -| M1 | **Document the bound**: Add to SECURITY.md: "AES-256-GCM with random nonces is safe for up to 2^32 encryptions per key. For higher volumes, rotate keys (M12) or use a counter-based nonce scheme." | ~0 LoC | Awareness | -| M2 | **Nonce counter option** (long-term): Add optional `nonceStrategy: 'random' \| 'counter'` to encryption options. Counter-based nonces guarantee uniqueness but require persistent state (a counter stored in the vault metadata). Random remains the default for simplicity. | ~60 LoC | Eliminates collision risk | -| M3 | **Key usage counter in vault**: Track `encryptionCount` in vault metadata. When it exceeds 2^31, emit a warning via observability: "Key has been used for N encryptions. Consider rotating." | ~20 LoC | Proactive warning | - -**Recommended**: M1 (immediate, zero-cost) + M3 (proactive warning). M2 is a significant design change that adds state management complexity — only needed for extremely high-volume use cases. - -**Defensive Tests** - -```js -describe('Concern 7: Nonce uniqueness', () => { - it('generates unique nonces across 1000 consecutive encryptions', ...); - it('nonce is exactly 12 bytes (96 bits)', ...); - it('different encryptions of same plaintext with same key produce different ciphertexts', ...); - it('vault tracks encryptionCount and increments per store', ...); - it('warns via observability when encryptionCount exceeds threshold', ...); -}); -``` +### M20 — Sentinel (`v5.6.0`) ---- +**Theme:** vault health, crypto hygiene, and safety workflows. -## Concern 8: Crypto Adapter Liskov Substitution Violation ✅ MITIGATED +Deliverables: -**Source:** CODE-EVAL.md, Flaw 1 +- `git cas vault status` +- `git cas gc` +- `encryptionCount` auto-rotation policy +- `.casrc` KDF parameter tuning with safe validation +- Human CLI warnings for nonce budget and KDF health +- Agent CLI warnings/results for the same health signals -**Status:** Task 16.1 implemented. All adapters: async `encryptBuffer`, `_validateKey` in `decryptBuffer`, `STREAM_NOT_CONSUMED` guard. Conformance test suite. +### M21 — Atelier (`v5.7.0`) -**The Problem** +**Theme:** vault ergonomics and publishing workflows. -The three `CryptoPort` implementations (Node, Bun, Web) differ in observable behavior: +Deliverables: -1. `NodeCryptoAdapter.encryptBuffer()` is synchronous (returns plain object), while Bun and Web return `Promise`. -2. `BunCryptoAdapter.decryptBuffer()` calls `_validateKey(key)` before decryption; Node and Web do not — the invalid key hits `node:crypto` directly, producing a less informative error. -3. `NodeCryptoAdapter.createEncryptionStream()` has no premature-finalize guard. Calling `finalize()` before consuming the stream returns garbage metadata on Node, but throws a clear `CasError('STREAM_NOT_CONSUMED')` on Bun and Deno. +- Named vaults +- `git cas vault add` to adopt existing trees +- Vault export flows: + - whole vault export + - single-entry export + - bulk export +- Publish flows: + - publish to working tree + - publish to branch + - auto-publish hook support +- File-level `--passphrase` CLI for standalone encrypted store flows -M15 Prism fixed the `sha256()` async inconsistency but left these three discrepancies untouched. +### M22 — Cartographer (`v5.8.0`) -**Mitigation:** Task 16.1. +**Theme:** repo intelligence and artifact comparison. ---- +Deliverables: -## Concern 9: FixedChunker Quadratic Buffer Allocation ✅ MITIGATED +- Duplicate-detection warnings during store +- `git cas scan` / dedup advisor +- Manifest diff engine +- Machine diff stream for the agent CLI +- Human compare view layered on the M19 shell -**Source:** CODE-EVAL.md, Flaw 4 +### M23 — Courier (`v5.9.0`) -**Status:** Task 16.4 implemented. Pre-allocated `Buffer.allocUnsafe(chunkSize)` working buffer. +**Theme:** artifact sets and transport. -**The Problem** +Deliverables: -`FixedChunker.chunk()` uses `Buffer.concat([buffer, data])` inside its async loop. Each call allocates a new buffer and copies the accumulated bytes. For a source yielding many small buffers (e.g., 4 KiB network reads into a 256 KiB chunk), this is O(n^2 / chunkSize) total byte copies. The CdcChunker, by contrast, uses a pre-allocated `Buffer.allocUnsafe(maxChunkSize)` with zero intermediate copies. +- Snapshot trees for directory-level store and restore +- Portable bundles for air-gap transfer +- Watch mode built on snapshot-root semantics rather than ad hoc per-file state -**Mitigation:** Task 16.4. +### M24 — Spectrum (`v5.10.0`) ---- +**Theme:** storage and observability extensibility. -## Concern 10: CDC Deduplication Defeated by Encrypt-Then-Chunk ✅ MITIGATED +Deliverables: -**Source:** CODE-EVAL.md, Flaw 5 +- `CompressionPort` +- Additional codecs: `zstd`, `brotli`, `lz4` +- Prometheus/OpenTelemetry adapter for `ObservabilityPort` -**Status:** Task 16.5 implemented. Runtime warning when encryption + CDC combined. +### M25 — Bastion (`v5.11.0`) -**The Problem** +**Theme:** enterprise key-management research with hard exit criteria. -Encryption is applied to the source stream *before* chunking. AES-GCM ciphertext is pseudorandom — identical plaintext produces different ciphertext (different random nonce each time). This means content-defined chunking (CDC) provides **zero deduplication benefit** for encrypted files. Users who combine `recipients` (or `encryptionKey`) with `chunking: { strategy: 'cdc' }` get CDC's computational overhead without its primary value proposition. +Deliverables: -This is a fundamental architectural constraint of the encrypt-then-chunk design. The alternative (chunk-then-encrypt) would require per-chunk nonces and auth tags, significantly complicating the manifest schema. This is documented as a known limitation, not a fixable bug. +- ADR for external key-management support +- Threat model for HSM/Vault-backed key flows +- Proof-of-concept `KeyManagementPort` adapter +- Decision memo on whether enterprise key management should become a product milestone -**Mitigation:** Task 16.5 (runtime warning + documentation). +## Delivery Standards ---- +Every planned milestone follows the repository release discipline: -## Summary Table +- Human CLI/TUI behavior remains backward compatible unless a release explicitly declares otherwise. +- The human `--json` flag remains convenience output, not the automation contract. +- The first machine interface release is JSONL-only and one-shot; no session protocol is planned before the contract proves useful. +- `agent restore` writes to the filesystem in v1; binary payloads do not share protocol `stdout`. +- Any user-visible feature added after M18 must include: + - at least one human CLI/TUI test, and + - at least one agent-protocol test when the feature is exposed to the machine surface. -| # | Type | Severity | Fix Cost | Recommended Action | Task | Status | -|---|------|----------|----------|--------------------|------|--------| -| C1 | Memory amplification | High | ~20 LoC | Add `maxRestoreBufferSize` guard | **16.2** | ✅ Done | -| C2 | Orphaned blobs | Medium | ~20 LoC | Report orphaned blob OIDs in error meta | **16.10** | ✅ Done | -| C3 | No chunk size cap | Medium | ~6 LoC | Enforce 100 MiB maximum | **16.6** | ✅ Done | -| C4 | Web Crypto buffering | Medium | ~15 LoC | Add buffer size guard in WebCryptoAdapter | **16.3** | ✅ Done | -| C5 | Passphrase exposure | High | ~90 LoC | Interactive prompt + file-based input | **16.11** | ✅ Done | -| C6 | KDF no rate limit | Low | ~10 LoC | Observability metric + CLI delay | **16.12** | ✅ Done | -| C7 | GCM nonce collision | Low | ~20 LoC | Document bound + vault usage counter | **16.13** | ✅ Done | -| C8 | Crypto adapter LSP violation | Medium | ~50 LoC | Normalize validation + finalize guards | **16.1** | ✅ Done | -| C9 | FixedChunker quadratic alloc | Low | ~20 LoC | Pre-allocated buffer | **16.4** | ✅ Done | -| C10 | Encrypt-then-chunk dedup loss | Medium | ~10 LoC | Runtime warning + documentation | **16.5** | ✅ Done | +## Document Boundaries -| # | Type | Theme | Est. Cost | Status | -|---|------|-------|-----------|--------| -| V1 | Feature | Snapshot trees (directory store) | ~410 LoC, ~19h | 🔲 Open | -| V2 | Feature | Portable bundles (air-gap transfer) | ~340 LoC, ~15h | 🔲 Open | -| V3 | Feature | Manifest diff engine | ~180 LoC, ~8h | 🔲 Open | -| V4 | Feature | CompressionPort + zstd/brotli/lz4 | ~180 LoC, ~8h | 🔲 Open | -| V5 | Feature | Watch mode (continuous sync) | ~220 LoC, ~10h | 🔲 Open | -| V6 | Feature | Interactive passphrase prompt | ~90 LoC, ~4h | ✅ Done — subsumed by **16.11** | -| V7 | Feature | Prometheus/OpenTelemetry ObservabilityPort adapter | ~150 LoC, ~6h | 🔲 Open | -| V8 | Feature | `encryptionCount` auto-rotation | ~120 LoC, ~5h | 🔲 Open | -| V9 | Feature | `vault status` command — show metadata, `encryptionCount`, entry count, nonce health | ~60 LoC, ~2h | 🔲 Open | -| V10 | Feature | `gc` command — `collectReferencedChunks` + `git gc` for orphan cleanup | ~80 LoC, ~4h | 🔲 Open | -| V11 | Feature | KDF parameter tuning via `.casrc` — `kdf.iterations`, `kdf.cost`, `kdf.blockSize`, `kdf.parallelization` with validation (reject insecure values below OWASP minimums) | ~40 LoC, ~2h | 🔲 Open | -| V12 | Feature | File-level passphrase CLI — `--passphrase` flag for standalone encrypted store without vault encryption. Library already supports `passphrase` + `kdfOptions` on `store()`/`storeFile()`. | ~30 LoC, ~1h | 🔲 Open | +- [ROADMAP.md](./ROADMAP.md): current reality plus future sequence +- [STATUS.md](./STATUS.md): compact project snapshot +- [COMPLETED_TASKS.md](./COMPLETED_TASKS.md): shipped milestone details +- [GRAVEYARD.md](./GRAVEYARD.md): superseded or merged-away work +- [CHANGELOG.md](./CHANGELOG.md): release-by-release history diff --git a/STATUS.md b/STATUS.md index 4be4428..345b6b9 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,88 +1,80 @@ # @git-stunts/cas — Project Status -**Current version:** v5.1.0 (Locksmith) -**Last release:** 2026-02-28 -**Test suite:** 757 tests (vitest) +**Current version:** `v5.3.1` +**Last release:** `2026-03-15` +**Current line:** M16 Capstone shipped in `v5.3.0`; `v5.3.1` is the maintenance follow-up that fixed repeated-chunk tree emission. **Runtimes:** Node.js 22.x, Bun, Deno --- -## What's shipped +## Interface Strategy -| Version | Codename | Highlights | -|---------|----------|------------| -| v5.1.0 | Locksmith | Envelope encryption (DEK/KEK), multi-recipient APIs, `--recipient` CLI, recipient management | -| v5.0.0 | Hydra | Content-defined chunking (CDC), `ChunkingPort`, buzhash engine, 98% dedup on edits | -| v4.0.1 | Spit Shine + Cockpit | CryptoPort refactor, `verify` command, `--json` mode, `runAction`, vault list filtering | -| v4.0.0 | Conduit | ObservabilityPort, `restoreStream()`, parallel chunk I/O, `concurrency` option | -| v3.1.0 | Bijou | Interactive vault dashboard, animated progress bars, `git cas inspect`, chunk heatmap | -| v3.0.0 | Vault | GC-safe ref-based storage (`refs/cas/vault`), slug-based addressing, vault CLI | -| v2.0.0 | Horizon | Compression (gzip), KDF (pbkdf2/scrypt), Merkle manifests | -| v1.x | — | Core CAS, AES-256-GCM encryption, fixed chunking, Git ODB persistence | +- **Human CLI/TUI:** the current public operator surface. Existing `git cas ...` commands, Bijou formatting, prompts, dashboards, and `--json` convenience output stay here. +- **Agent CLI:** planned next as `git cas agent`. It will be JSONL-first, non-interactive by default, and independent from Bijou rendering or TTY-only behavior. --- -## What's next +## Recently Shipped -One open milestone remains. +| Version | Milestone | Highlights | +|---------|-----------|------------| +| `v5.3.1` | Maintenance | Repeated-chunk tree integrity fix; unique chunk tree entries; `git fsck` regression coverage | +| `v5.3.0` | M16 Capstone | Audit remediation, `.casrc`, passphrase-file support, restore guards, `encryptionCount`, lifecycle rename | +| `v5.2.0` | M12 Carousel | Key rotation without re-encrypting data | +| `v5.1.0` | M11 Locksmith | Envelope encryption and recipient management | +| `v5.0.0` | M10 Hydra | Content-defined chunking | +| `v4.0.1` | M8 + M9 | Review hardening, `verify`, `--json`, CLI polish | +| `v4.0.0` | M14 Conduit | Streaming restore, observability, parallel chunk I/O | +| `v3.1.0` | M13 Bijou | Interactive dashboard and animated progress | -### M12 — Carousel (~13h) -Key rotation without re-encrypting data. Now unblocked by M11 Locksmith. +--- -- [ ] **12.1** Key rotation workflow (`rotateKey()`) -- [ ] **12.2** Key version tracking in manifest -- [ ] **12.3** CLI key rotation commands -- [ ] **12.4** Vault-level key rotation +## Next Up ---- +### M17 — Ledger (`v5.3.2`) + +Planning and ops reset: + +- Reconcile `ROADMAP.md`, `STATUS.md`, and release messaging +- Add review automation (`CODEOWNERS` or equivalent) +- Document Git tree ordering test conventions +- Define release-prep workflow for changelog/version timing +- Add property-based fuzz coverage for envelope encryption + +### M18 — Relay (`v5.4.0`) -## Dependency graph +LLM-native CLI foundation: -``` -M8 Spit Shine ──────── ✅ v4.0.1 -M9 Cockpit ─────────── ✅ v4.0.1 -M10 Hydra ──────────── ✅ v5.0.0 -M11 Locksmith ──────── ✅ v5.1.0 - └──► M12 Carousel ── (ready) -``` +- Introduce `git cas agent` +- Define the JSONL protocol envelope and exit codes +- Add machine-facing parity for the current operational command set +- Enforce strict non-interactive input handling + +### M19 — Nouveau (`v5.5.0`) + +Human UX refresh: + +- Upgrade Bijou packages to `3.0.0` +- Move the inspector shell to the v3 `ViewOutput` model +- Split the dashboard into sub-apps +- Add better styling, motion, layout persistence, and richer heatmap/detail rendering --- -## Backlog (unscheduled ideas) - -- Named vaults (`refs/cas/vaults/`) -- Export vault to archive -- Publish to working tree / branch -- Duplicate detection on store -- Repo scan / dedup advisor -- Add `CODEOWNERS` or reviewer auto-assignment for PRs -- Document Git tree filename ordering semantics in test conventions -- Define release-prep workflow for CHANGELOG/version bump timing -- Automate test count injection into CHANGELOG from CI output -- Property-based fuzz tests for envelope encryption round-trips -- Investigate HSM/Vault key management as a future `KeyManagementPort` - -## Visions (researched, not committed) - -- **V1** Snapshot trees — directory-level store (~410 LoC, ~19h) -- **V2** Portable bundles — air-gap transfer (~340 LoC, ~15h) -- **V3** Manifest diff engine (~180 LoC, ~8h) -- **V4** CompressionPort — zstd, brotli, lz4 (~180 LoC, ~8h) -- **V5** Watch mode — continuous sync (~220 LoC, ~10h) -- **V6** Interactive passphrase prompt (~90 LoC, ~4h) - -## Known concerns - -| # | Issue | Severity | Summary | -|---|-------|----------|---------| -| C1 | Memory amplification | High | Encrypted/compressed restore buffers entire file | -| C2 | Orphaned blobs | Medium | STREAM_ERROR leaves unreferenced blobs in ODB | -| C3 | No chunk size cap | Medium | No upper bound on configured chunk size | -| C4 | Web Crypto buffering | Medium | Deno adapter silently buffers entire file | -| C5 | Passphrase exposure | High | `--vault-passphrase` visible in shell history | -| C6 | KDF no rate limit | Low | No brute-force detection on failed decryption | -| C7 | GCM nonce collision | Low | 96-bit random nonce, safe to ~2^32 encryptions | +## Sequenced Roadmap + +| Version | Milestone | Theme | +|---------|-----------|-------| +| `v5.3.2` | M17 Ledger | Planning and ops reset | +| `v5.4.0` | M18 Relay | LLM-native CLI foundation | +| `v5.5.0` | M19 Nouveau | Bijou v3 human UX refresh | +| `v5.6.0` | M20 Sentinel | Vault health and safety | +| `v5.7.0` | M21 Atelier | Vault ergonomics and publishing | +| `v5.8.0` | M22 Cartographer | Repo intelligence and change analysis | +| `v5.9.0` | M23 Courier | Artifact sets and transfer | +| `v5.10.0` | M24 Spectrum | Storage and observability extensibility | +| `v5.11.0` | M25 Bastion | Enterprise key-management research | --- -*Full task cards: [ROADMAP.md](./ROADMAP.md) | Completed: [COMPLETED_TASKS.md](./COMPLETED_TASKS.md) | Superseded: [GRAVEYARD.md](./GRAVEYARD.md)* +*Future details: [ROADMAP.md](./ROADMAP.md) | Shipped detail: [COMPLETED_TASKS.md](./COMPLETED_TASKS.md) | Superseded: [GRAVEYARD.md](./GRAVEYARD.md)* diff --git a/test/unit/domain/services/rotateVaultPassphrase.test.js b/test/unit/domain/services/rotateVaultPassphrase.test.js index a16c9cc..f5c1fc2 100644 --- a/test/unit/domain/services/rotateVaultPassphrase.test.js +++ b/test/unit/domain/services/rotateVaultPassphrase.test.js @@ -15,6 +15,8 @@ import { getTestCryptoAdapter } from '../../../helpers/crypto-adapter.js'; import rotateVaultPassphrase from '../../../../src/domain/services/rotateVaultPassphrase.js'; import CasError from '../../../../src/domain/errors/CasError.js'; +const LONG_TEST_TIMEOUT_MS = 15000; + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -109,7 +111,7 @@ describe('rotateVaultPassphrase – 3 envelope entries', () => { const { buffer } = await service.restore({ manifest, encryptionKey: newKey }); expect(buffer.equals(originals[name])).toBe(true); } - }); + }, LONG_TEST_TIMEOUT_MS); }); describe('rotateVaultPassphrase – mixed entries', () => { @@ -153,7 +155,7 @@ describe('rotateVaultPassphrase – mixed entries', () => { expect(rotatedSlugs.sort()).toEqual(['env1', 'env2']); expect(skippedSlugs).toEqual(['direct']); - }); + }, LONG_TEST_TIMEOUT_MS); }); describe('rotateVaultPassphrase – error cases', () => { @@ -175,7 +177,7 @@ describe('rotateVaultPassphrase – error cases', () => { await expect( rotateVaultPassphrase({ service, vault }, { oldPassphrase: 'wrong', newPassphrase: 'new' }), ).rejects.toThrow(); - }); + }, LONG_TEST_TIMEOUT_MS); it('vault not encrypted → VAULT_METADATA_INVALID', async () => { await vault.initVault(); @@ -188,7 +190,7 @@ describe('rotateVaultPassphrase – error cases', () => { await expect( rotateVaultPassphrase({ service, vault }, { oldPassphrase: 'any', newPassphrase: 'new' }), ).rejects.toMatchObject({ code: 'VAULT_METADATA_INVALID' }); - }); + }, LONG_TEST_TIMEOUT_MS); }); describe('rotateVaultPassphrase – KDF options', () => { @@ -218,7 +220,7 @@ describe('rotateVaultPassphrase – KDF options', () => { const newState = await vault.readState(); expect(newState.metadata.encryption.kdf.algorithm).toBe('scrypt'); - }); + }, LONG_TEST_TIMEOUT_MS); it('metadata updated with new KDF salt', async () => { const oldPass = 'old-pass'; @@ -237,7 +239,7 @@ describe('rotateVaultPassphrase – KDF options', () => { const newState = await vault.readState(); expect(newState.metadata.encryption.kdf.salt).not.toBe(oldSalt); expect(newState.metadata.encryption.kdf.algorithm).toBe(oldState.metadata.encryption.kdf.algorithm); - }); + }, LONG_TEST_TIMEOUT_MS); }); describe('rotateVaultPassphrase – retry success', () => { @@ -274,7 +276,7 @@ describe('rotateVaultPassphrase – retry success', () => { ); expect(result.commitOid).toMatch(/^[0-9a-f]{40}$/); expect(calls).toBe(2); - }); + }, LONG_TEST_TIMEOUT_MS); }); describe('rotateVaultPassphrase – maxRetries exhausted', () => { @@ -309,7 +311,7 @@ describe('rotateVaultPassphrase – maxRetries exhausted', () => { ), ).rejects.toMatchObject({ code: 'VAULT_CONFLICT' }); expect(calls).toBe(1); - }); + }, LONG_TEST_TIMEOUT_MS); }); describe('rotateVaultPassphrase – default retry count', () => { @@ -344,5 +346,5 @@ describe('rotateVaultPassphrase – default retry count', () => { ), ).rejects.toMatchObject({ code: 'VAULT_CONFLICT' }); expect(calls).toBe(3); - }); + }, LONG_TEST_TIMEOUT_MS); }); diff --git a/test/unit/facade/ContentAddressableStore.rotation.test.js b/test/unit/facade/ContentAddressableStore.rotation.test.js index 4c5ac4c..ca1beb1 100644 --- a/test/unit/facade/ContentAddressableStore.rotation.test.js +++ b/test/unit/facade/ContentAddressableStore.rotation.test.js @@ -7,6 +7,8 @@ import { execSync } from 'node:child_process'; import GitPlumbing from '@git-stunts/plumbing'; import ContentAddressableStore from '../../../index.js'; +const LONG_TEST_TIMEOUT_MS = 15000; + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -72,5 +74,5 @@ describe('ContentAddressableStore – rotateVaultPassphrase (wiring)', () => { expect(commitOid).toMatch(/^[0-9a-f]{40}$/); expect(rotatedSlugs).toEqual(['asset']); expect(skippedSlugs).toEqual([]); - }); + }, LONG_TEST_TIMEOUT_MS); }); diff --git a/test/unit/vault/VaultService.test.js b/test/unit/vault/VaultService.test.js index 93d3697..865777c 100644 --- a/test/unit/vault/VaultService.test.js +++ b/test/unit/vault/VaultService.test.js @@ -2,6 +2,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import VaultService from '../../../src/domain/services/VaultService.js'; import CasError from '../../../src/domain/errors/CasError.js'; +const LONG_TEST_TIMEOUT_MS = 15000; + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -686,5 +688,5 @@ describe('ContentAddressableStore vault delegation', () => { it('VAULT_REF matches VaultService', async () => { const { default: ContentAddressableStore } = await import('../../../index.js'); expect(ContentAddressableStore.VAULT_REF).toBe(VaultService.VAULT_REF); - }); + }, LONG_TEST_TIMEOUT_MS); }); From c3e72710fa946410ad490fede7146b373ebdc032 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 15 Mar 2026 14:54:52 -0700 Subject: [PATCH 2/6] Fix: stabilize multi-runtime CLI test runs --- .dockerignore | 1 + bin/actions.js | 9 ++- bin/git-cas.js | 5 +- bin/io.js | 86 ++++++++++++++++++++++ test/integration/round-trip.test.js | 14 +++- test/integration/vault-cli.test.js | 109 +++++++++++++++++++--------- test/integration/vault.test.js | 16 +++- test/unit/cli/actions.test.js | 42 ++++++----- test/unit/cli/io.test.js | 61 ++++++++++++++++ vitest.config.js | 11 +++ 10 files changed, 292 insertions(+), 62 deletions(-) create mode 100644 bin/io.js create mode 100644 test/unit/cli/io.test.js create mode 100644 vitest.config.js diff --git a/.dockerignore b/.dockerignore index 29e140f..6ee13a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +.git node_modules npm-debug.log .gitignore diff --git a/bin/actions.js b/bin/actions.js index 2a28ce4..35dea70 100644 --- a/bin/actions.js +++ b/bin/actions.js @@ -70,10 +70,13 @@ function defaultDelay(ms) { * * @param {(...args: any[]) => Promise} fn - The async action function. * @param {() => boolean} getJson - Lazy getter for --json flag value. - * @param {{ delay?: (ms: number) => Promise }} [options] - Injectable dependencies. + * @param {{ delay?: (ms: number) => Promise, setExitCode?: (code: number) => void }} [options] - Injectable dependencies. * @returns {(...args: any[]) => Promise} Wrapped action. */ -export function runAction(fn, getJson, { delay = defaultDelay } = {}) { +export function runAction(fn, getJson, { + delay = defaultDelay, + setExitCode = (code) => { process.exitCode = code; }, +} = {}) { return async (/** @type {any[]} */ ...args) => { try { await fn(...args); @@ -81,8 +84,8 @@ export function runAction(fn, getJson, { delay = defaultDelay } = {}) { if (err?.code === 'INTEGRITY_ERROR') { await delay(1000); } + setExitCode(1); writeError(err, getJson()); - process.exitCode = 1; } }; } diff --git a/bin/git-cas.js b/bin/git-cas.js index d034386..6f6b3ca 100755 --- a/bin/git-cas.js +++ b/bin/git-cas.js @@ -11,11 +11,13 @@ import { renderHistoryTimeline } from './ui/history-timeline.js'; import { renderManifestView } from './ui/manifest-view.js'; import { renderHeatmap } from './ui/heatmap.js'; import { runAction } from './actions.js'; +import { flushStdioAndExit, installBrokenPipeHandlers } from './io.js'; import { filterEntries, formatTable, formatTabSeparated } from './ui/vault-list.js'; import { readPassphraseFile, promptPassphrase } from './ui/passphrase-prompt.js'; import { loadConfig, mergeConfig } from './config.js'; const getJson = () => program.opts().json; +installBrokenPipeHandlers(); program .name('git-cas') @@ -751,5 +753,4 @@ await program.parseAsync(); // Flush stdout/stderr before exiting — spawned git child processes leave // libuv handles that prevent natural exit in containerized environments. -const code = process.exitCode || 0; -process.stdout.write('', () => process.stderr.write('', () => process.exit(code))); +await flushStdioAndExit(); diff --git a/bin/io.js b/bin/io.js new file mode 100644 index 0000000..e8a09d6 --- /dev/null +++ b/bin/io.js @@ -0,0 +1,86 @@ +import { setTimeout as delay } from 'node:timers/promises'; + +/** + * @param {unknown} err + * @returns {err is NodeJS.ErrnoException} + */ +function isBrokenPipeError(err) { + return Boolean(err && typeof err === 'object' && /** @type {NodeJS.ErrnoException} */ (err).code === 'EPIPE'); +} + +/** + * Install stdout/stderr error handlers that exit cleanly when the downstream + * consumer closes the pipe before the CLI finishes writing. + * + * @param {Object} [options] + * @param {{ on(event: string, listener: (...args: any[]) => void): any, removeListener(event: string, listener: (...args: any[]) => void): any }} [options.stdout] + * @param {{ on(event: string, listener: (...args: any[]) => void): any, removeListener(event: string, listener: (...args: any[]) => void): any }} [options.stderr] + * @param {(code?: number) => never} [options.exit] + * @param {() => number} [options.getExitCode] + * @returns {{ dispose(): void }} + */ +export function installBrokenPipeHandlers({ + stdout = process.stdout, + stderr = process.stderr, + exit = process.exit, + getExitCode = () => process.exitCode || 0, +} = {}) { + const onError = (/** @type {unknown} */ err) => { + if (isBrokenPipeError(err)) { + exit(getExitCode()); + } + }; + + stdout.on('error', onError); + stderr.on('error', onError); + + return { + dispose() { + stdout.removeListener('error', onError); + stderr.removeListener('error', onError); + }, + }; +} + +/** + * Flush stdout/stderr before exit so the CLI does not hang on open handles in + * containerized test environments. + * + * @param {Object} [options] + * @param {{ write(chunk: string, callback?: () => void): boolean }} [options.stdout] + * @param {{ write(chunk: string, callback?: () => void): boolean }} [options.stderr] + * @param {(code?: number) => void} [options.exit] + * @param {number} [options.code] + * @returns {Promise} + */ +export async function flushStdioAndExit({ + stdout = process.stdout, + stderr = process.stderr, + exit = process.exit, + code = process.exitCode || 0, +} = {}) { + await flushStream(stdout); + await flushStream(stderr); + exit(code); +} + +/** + * @param {{ write(chunk: string, callback?: () => void): boolean }} stream + * @returns {Promise} + */ +async function flushStream(stream) { + try { + await new Promise((resolve) => { + stream.write('', resolve); + }); + } catch (err) { + if (!isBrokenPipeError(err)) { + throw err; + } + } + + // Give stream error handlers one turn to observe late EPIPE events before exit. + await delay(0); +} + +export { isBrokenPipeError }; diff --git a/test/integration/round-trip.test.js b/test/integration/round-trip.test.js index 479423b..e2eeee0 100644 --- a/test/integration/round-trip.test.js +++ b/test/integration/round-trip.test.js @@ -10,7 +10,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'; import { randomBytes } from 'node:crypto'; -import { execSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import path from 'node:path'; import os from 'node:os'; import GitPlumbing from '@git-stunts/plumbing'; @@ -31,9 +31,19 @@ let repoDir; let cas; let casCbor; +function initBareRepo(cwd) { + const result = spawnSync('git', ['init', '--bare'], { cwd, encoding: 'utf8' }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error(`${result.stderr ?? result.stdout ?? 'git init --bare failed'}`.trim()); + } +} + beforeAll(() => { repoDir = mkdtempSync(path.join(os.tmpdir(), 'cas-integ-')); - execSync('git init --bare', { cwd: repoDir, stdio: 'ignore' }); + initBareRepo(repoDir); const plumbing = GitPlumbing.createDefault({ cwd: repoDir }); cas = new ContentAddressableStore({ plumbing }); diff --git a/test/integration/vault-cli.test.js b/test/integration/vault-cli.test.js index 603f4b7..b002ea0 100644 --- a/test/integration/vault-cli.test.js +++ b/test/integration/vault-cli.test.js @@ -11,7 +11,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'; import { randomBytes } from 'node:crypto'; -import { execSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import path from 'node:path'; import os from 'node:os'; import { fileURLToPath } from 'node:url'; @@ -36,19 +36,58 @@ const BIN = path.resolve(__dirname, '../../bin/git-cas.js'); * - Node → node