feat: x402 marketplace + architecture review bundle (#513-#535)

.github/release-template.md

@@ -96,6 +96,19 @@ repositories or docs.]
 ## Breaking changes / Migration notes
 
 - [Delete this section if there are no breaking changes.]
+- **Pre-release tester warning**: If you ran an unreleased marketplace or
+  chart-consolidation branch before this release, `obol stack up` may fail
+  with Helm `invalid ownership metadata` errors for resources or namespaces
+  that moved into the `base` chart. This is not a supported production
+  migration path. Back up anything you need from the local test stack, then
+  recreate it:
+
+  ```bash
+  obol stack down
+  obol stack purge --force
+  obol stack init
+  obol stack up
+  ```
 
 ## Known issues
 

.github/workflows/helm-template-smoke.yml

@@ -0,0 +1,116 @@
+name: Helm Template Smoke
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'internal/embed/infrastructure/**'
+      - '.github/workflows/helm-template-smoke.yml'
+  push:
+    branches: [ main ]
+    paths:
+      - 'internal/embed/infrastructure/**'
+      - '.github/workflows/helm-template-smoke.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  helm-template-smoke:
+    name: helm template embedded chart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+      - name: Set up Helm
+        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
+        with:
+          version: v3.20.1   # match obolup.sh pinned version
+
+      - name: helm template ./base
+        run: |
+          # Render the embedded `base` chart and fail on Go-template parse
+          # errors. Catches bugs like the unescaped `{{ $labels }}` in
+          # PrometheusRule annotations that broke `helm upgrade base` on
+          # every `obol stack up` (see PR #527). `go test ./...` does not
+          # exercise Helm rendering, so this is the only pre-merge gate
+          # for chart parse errors.
+          #
+          # The base chart contains `{{PLACEHOLDER}}` strings (e.g.
+          # `{{OLLAMA_HOST_IP}}`, `{{CLUSTER_ID}}`) that are substituted
+          # by `internal/defaults/defaults.go::InfrastructureReplacements`
+          # before helmfile runs. Helm's Go-template parser would treat
+          # them as actions and fail, so we substitute stub values into
+          # a working copy first — mirroring what `obol stack init` does.
+          set -euo pipefail
+          workdir="$(mktemp -d)"
+          cp -R internal/embed/infrastructure/base "$workdir/base"
+          # Mirror internal/defaults InfrastructureReplacements with CI stubs.
+          find "$workdir/base" -type f -name '*.yaml' -print0 \
+            | xargs -0 sed -i \
+              -e 's/{{OLLAMA_HOST_IP}}/127.0.0.1/g' \
+              -e 's/{{OLLAMA_HOST}}/localhost/g' \
+              -e 's/{{CLUSTER_ID}}/ci-helm-smoke/g'
+          # Match values passed by helmfile.yaml `releases[base]`.
+          helm template base "$workdir/base" \
+            --set dataDir=/data \
+            --set network=mainnet \
+            > "$workdir/base-rendered.yaml"
+
+          # Kubernetes object identity must be unique within one rendered
+          # chart. Helm will happily render duplicate apiVersion/kind/name
+          # tuples and leave the actual outcome to manifest ordering; this
+          # caught the duplicated obol-frontend ClusterRole/Binding review bug.
+          awk '
+          function flush() {
+            if (api && kind && name) {
+              key = api "/" kind "/" ns "/" name
+              count[key]++
+            }
+            api = kind = name = ns = ""; inmeta = 0
+          }
+          /^---/ { flush(); next }
+          /^apiVersion:/ { api = $2; next }
+          /^kind:/ { kind = $2; next }
+          /^metadata:/ { inmeta = 1; next }
+          inmeta && /^  name:/ { name = $2; next }
+          inmeta && /^  namespace:/ { ns = $2; next }
+          /^[^ ]/ && $0 !~ /^(apiVersion|kind|metadata):/ { inmeta = 0 }
+          END {
+            flush()
+            for (k in count) {
+              if (count[k] > 1) {
+                print count[k] " " k
+                dup = 1
+              }
+            }
+            exit dup
+          }' "$workdir/base-rendered.yaml"
+
+      - name: helm template ./cloudflared
+        run: |
+          # The cloudflared chart has no placeholder substitution and uses
+          # default values from values.yaml.
+          set -euo pipefail
+          helm template cloudflared internal/embed/infrastructure/cloudflared \
+            > /dev/null
+
+      - name: helm lint ./base
+        run: |
+          set -euo pipefail
+          workdir="$(mktemp -d)"
+          cp -R internal/embed/infrastructure/base "$workdir/base"
+          find "$workdir/base" -type f -name '*.yaml' -print0 \
+            | xargs -0 sed -i \
+              -e 's/{{OLLAMA_HOST_IP}}/127.0.0.1/g' \
+              -e 's/{{OLLAMA_HOST}}/localhost/g' \
+              -e 's/{{CLUSTER_ID}}/ci-helm-smoke/g'
+          helm lint "$workdir/base" \
+            --set dataDir=/data \
+            --set network=mainnet
+
+      - name: helm lint ./cloudflared
+        run: |
+          set -euo pipefail
+          helm lint internal/embed/infrastructure/cloudflared

.github/workflows/lint-test.yaml

@@ -4,6 +4,9 @@ on:
   pull_request:
     branches: [ main ]
 
+permissions:
+  contents: read
+
 jobs:
   lint-test:
     runs-on: ubuntu-latest
@@ -43,3 +46,31 @@ jobs:
 
       - name: Run chart-testing (install)
         run: ct install --target-branch ${{ github.event.repository.default_branch }}
+
+  generate-check:
+    name: CRD generation up-to-date
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+      - name: Set up Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Set up just
+        uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.2
+
+      - name: Regenerate CRDs + DeepCopy
+        run: just generate
+
+      - name: Fail if regeneration changed any tracked files
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "::error::CRD manifests or DeepCopy methods are out of date."
+            echo "::error::Run 'just generate' locally and commit the result."
+            git status
+            git --no-pager diff
+            exit 1
+          fi

CLAUDE.md

@@ -342,6 +342,8 @@ A registry digest pin instead of `:latest` on the verifier means your dev rewrit
 
 For a fuller debug catalog with symptom→fix mapping, see `.agents/skills/obol-stack-dev/references/release-smoke-debugging.md`.
 
+For observability architecture decisions (Prometheus retention vs. on-chain canonical record, counter-reset semantics, recording-rule naming, label conventions, CRD versioning stance, `clamp_min` epsilon), see `docs/observability.md` — read this before adding a new metric, recording rule, or proposing counter persistence.
+
 ### Security: Tunnel Exposure
 
 The Cloudflare tunnel exposes the cluster to the public internet. Only x402-gated endpoints and discovery metadata should be reachable via the tunnel hostname. Internal services (frontend, eRPC, LiteLLM, monitoring) MUST have `hostnames: ["obol.stack"]` on their HTTPRoutes to restrict them to local access.

Dockerfile.serviceoffer-controller

@@ -5,6 +5,6 @@ RUN go mod download
 COPY . .
 RUN CGO_ENABLED=0 go build -o /serviceoffer-controller ./cmd/serviceoffer-controller
 
-FROM gcr.io/distroless/static-debian12
+FROM gcr.io/distroless/static-debian12:nonroot
 COPY --from=builder /serviceoffer-controller /serviceoffer-controller
 ENTRYPOINT ["/serviceoffer-controller"]

cmd/obol/sell.go

@@ -2331,15 +2331,42 @@ Examples:
 func sellStopCommand(cfg *config.Config) *cli.Command {
 	return &cli.Command{
 		Name:      "stop",
-		Usage:     "Pause a ServiceOffer without deleting it",
+		Usage:     "Drain a ServiceOffer gracefully (advertises wind-down via discovery, then tears down the route)",
 		ArgsUsage: "<name>",
+		Description: `Marks a ServiceOffer as draining. While draining:
+  - The offer stays in /skill.md and /.well-known/agent-registration.json
+    with available=false and a drainEndsAt timestamp, so external
+    discovery (and ERC-8004 reputation scorers) can see the wind-down.
+  - The HTTPRoute and x402 payment gate STAY UP for the grace period
+    so buyers can complete in-flight payments.
+  - When the grace period elapses, the controller tears down the route
+    and marks PaymentGateReady/RoutePublished False with reason=Drained.
+
+The ServiceOffer CR itself is preserved — use 'obol sell delete' to
+remove it entirely (which also tombstones the ERC-8004 record).
+
+Flags:
+  --grace 30m   Override the grace period (default 1h).
+  --force       Skip the drain window (equivalent to --grace 0). Use
+                this when the abrupt-teardown behavior of the old
+                pause annotation is required for behavior parity.`,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:     "namespace",
 				Aliases:  []string{"n"},
 				Usage:    "Namespace of the ServiceOffer",
 				Required: true,
 			},
+			&cli.DurationFlag{
+				Name:  "grace",
+				Usage: "Drain grace period (e.g. 30m, 2h). Defaults to 1h.",
+				Value: monetizeapi.DefaultDrainGracePeriod,
+			},
+			&cli.BoolFlag{
+				Name:    "force",
+				Aliases: []string{"now"},
+				Usage:   "Skip the drain window and tear the route down on the next reconcile (alias: --now)",
+			},
 		},
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			u := getUI(cmd)
@@ -2352,19 +2379,37 @@ func sellStopCommand(cfg *config.Config) *cli.Command {
 				return err
 			}
 			ns := cmd.String("namespace")
+			grace := cmd.Duration("grace")
+			if cmd.Bool("force") {
+				grace = 0
+			}
+			if grace < 0 {
+				return errors.New("--grace must be >= 0")
+			}
 
-			u.Infof("Stopping the service offering %s/%s...", ns, name)
-
-			removePricingRoute(cfg, u, name)
-
-			patchJSON := `{"status":{"conditions":[{"type":"Ready","status":"False","reason":"Stopped","message":"Offer stopped by user"}]}}`
-			err := kubectlRun(cfg, "patch", "serviceoffers.obol.org", name, "-n", ns,
-				"--type=merge", "-p", patchJSON)
-			if err != nil {
-				return fmt.Errorf("failed to pause serviceoffer: %w", err)
+			now := time.Now().UTC()
+			drainEndsAt := now.Add(grace)
+
+			// metav1.Duration JSON-marshals as the string form (e.g.
+			// "1h0m0s"), and metav1.Time marshals as RFC3339. We can
+			// emit a tiny strategic-merge patch directly without
+			// importing the meta types into the CLI.
+			patchJSON := fmt.Sprintf(
+				`{"spec":{"drainAt":%q,"drainGracePeriod":%q}}`,
+				now.Format(time.RFC3339),
+				grace.String(),
+			)
+			if err := kubectlRun(cfg, "patch", "serviceoffers.obol.org", name, "-n", ns,
+				"--type=merge", "-p", patchJSON); err != nil {
+				return fmt.Errorf("failed to drain serviceoffer: %w", err)
 			}
 
-			u.Successf("Service offering %s/%s stopped.", ns, name)
+			if grace == 0 {
+				u.Successf("ServiceOffer %s/%s draining; route will be removed on the next reconcile (--force).", ns, name)
+			} else {
+				u.Successf("ServiceOffer %s/%s draining; route will be removed at %s.", ns, name, drainEndsAt.Format(time.RFC3339))
+			}
+			u.Infof("In-flight buyers can complete payments until then. Run `obol sell delete %s -n %s` to fully remove.", name, ns)
 			return nil
 		},
 	}
@@ -2518,8 +2563,6 @@ func sellDeleteCommand(cfg *config.Config) *cli.Command {
 				}
 			}
 
-			removePricingRoute(cfg, u, name)
-
 			// Identity-level registration ownership lives in the AgentIdentity
 			// CR and is managed by the controller. The CLI no longer patches
 			// the registration ConfigMap here; deleting the ServiceOffer is
@@ -4126,7 +4169,3 @@ func manifestNSName(manifest map[string]any) (string, string) {
 	return ns, name
 }
 
-// removePricingRoute is a no-op retained for compatibility.
-// The serviceoffer-controller now manages pricing routes via the ServiceOffer
-// informer; static ConfigMap routes are no longer used.
-func removePricingRoute(_ *config.Config, _ *ui.UI, _ string) {}

cmd/obol/sell_test.go

@@ -626,9 +626,20 @@ func TestSellStop_Structure(t *testing.T) {
 	stop := findSubcommand(t, cmd, "stop")
 	flags := flagMap(stop)
 
-	requireFlags(t, flags, "namespace")
+	requireFlags(t, flags, "namespace", "grace", "force")
 	assertFlagRequired(t, flags, "namespace")
 	assertFlagHasAlias(t, flags, "namespace", "n")
+	// --now is the documented alias for --force; if it disappears,
+	// scripted operators that rely on it break silently.
+	assertFlagHasAlias(t, flags, "force", "now")
+
+	graceFlag, ok := flags["grace"].(*cli.DurationFlag)
+	if !ok {
+		t.Fatalf("--grace should be *cli.DurationFlag, got %T", flags["grace"])
+	}
+	if graceFlag.Value != monetizeapi.DefaultDrainGracePeriod {
+		t.Errorf("--grace default = %v, want %v", graceFlag.Value, monetizeapi.DefaultDrainGracePeriod)
+	}
 }
 
 func TestSellDelete_Structure(t *testing.T) {