Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 56 additions & 17 deletions cmd/obol/sell.go
Original file line number Diff line number Diff line change
Expand Up @@ -2331,15 +2331,42 @@ Examples:
func sellStopCommand(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "stop",
Usage: "Pause a ServiceOffer without deleting it",
Usage: "Drain a ServiceOffer gracefully (advertises wind-down via discovery, then tears down the route)",
ArgsUsage: "<name>",
Description: `Marks a ServiceOffer as draining. While draining:
- The offer stays in /skill.md and /.well-known/agent-registration.json
with available=false and a drainEndsAt timestamp, so external
discovery (and ERC-8004 reputation scorers) can see the wind-down.
- The HTTPRoute and x402 payment gate STAY UP for the grace period
so buyers can complete in-flight payments.
- When the grace period elapses, the controller tears down the route
and marks PaymentGateReady/RoutePublished False with reason=Drained.

The ServiceOffer CR itself is preserved — use 'obol sell delete' to
remove it entirely (which also tombstones the ERC-8004 record).

Flags:
--grace 30m Override the grace period (default 1h).
--force Skip the drain window (equivalent to --grace 0). Use
this when the abrupt-teardown behavior of the old
pause annotation is required for behavior parity.`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "namespace",
Aliases: []string{"n"},
Usage: "Namespace of the ServiceOffer",
Required: true,
},
&cli.DurationFlag{
Name: "grace",
Usage: "Drain grace period (e.g. 30m, 2h). Defaults to 1h.",
Value: monetizeapi.DefaultDrainGracePeriod,
},
&cli.BoolFlag{
Name: "force",
Aliases: []string{"now"},
Usage: "Skip the drain window and tear the route down on the next reconcile (alias: --now)",
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
u := getUI(cmd)
Expand All @@ -2352,19 +2379,37 @@ func sellStopCommand(cfg *config.Config) *cli.Command {
return err
}
ns := cmd.String("namespace")
grace := cmd.Duration("grace")
if cmd.Bool("force") {
grace = 0
}
if grace < 0 {
return errors.New("--grace must be >= 0")
}

u.Infof("Stopping the service offering %s/%s...", ns, name)

removePricingRoute(cfg, u, name)

patchJSON := `{"status":{"conditions":[{"type":"Ready","status":"False","reason":"Stopped","message":"Offer stopped by user"}]}}`
err := kubectlRun(cfg, "patch", "serviceoffers.obol.org", name, "-n", ns,
"--type=merge", "-p", patchJSON)
if err != nil {
return fmt.Errorf("failed to pause serviceoffer: %w", err)
now := time.Now().UTC()
drainEndsAt := now.Add(grace)

// metav1.Duration JSON-marshals as the string form (e.g.
// "1h0m0s"), and metav1.Time marshals as RFC3339. We can
// emit a tiny strategic-merge patch directly without
// importing the meta types into the CLI.
patchJSON := fmt.Sprintf(
`{"spec":{"drainAt":%q,"drainGracePeriod":%q}}`,
now.Format(time.RFC3339),
grace.String(),
)
if err := kubectlRun(cfg, "patch", "serviceoffers.obol.org", name, "-n", ns,
"--type=merge", "-p", patchJSON); err != nil {
return fmt.Errorf("failed to drain serviceoffer: %w", err)
}

u.Successf("Service offering %s/%s stopped.", ns, name)
if grace == 0 {
u.Successf("ServiceOffer %s/%s draining; route will be removed on the next reconcile (--force).", ns, name)
} else {
u.Successf("ServiceOffer %s/%s draining; route will be removed at %s.", ns, name, drainEndsAt.Format(time.RFC3339))
}
u.Infof("In-flight buyers can complete payments until then. Run `obol sell delete %s -n %s` to fully remove.", name, ns)
return nil
},
}
Expand Down Expand Up @@ -2518,8 +2563,6 @@ func sellDeleteCommand(cfg *config.Config) *cli.Command {
}
}

removePricingRoute(cfg, u, name)

// Identity-level registration ownership lives in the AgentIdentity
// CR and is managed by the controller. The CLI no longer patches
// the registration ConfigMap here; deleting the ServiceOffer is
Expand Down Expand Up @@ -4126,7 +4169,3 @@ func manifestNSName(manifest map[string]any) (string, string) {
return ns, name
}

// removePricingRoute is a no-op retained for compatibility.
// The serviceoffer-controller now manages pricing routes via the ServiceOffer
// informer; static ConfigMap routes are no longer used.
func removePricingRoute(_ *config.Config, _ *ui.UI, _ string) {}
13 changes: 12 additions & 1 deletion cmd/obol/sell_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -626,9 +626,20 @@ func TestSellStop_Structure(t *testing.T) {
stop := findSubcommand(t, cmd, "stop")
flags := flagMap(stop)

requireFlags(t, flags, "namespace")
requireFlags(t, flags, "namespace", "grace", "force")
assertFlagRequired(t, flags, "namespace")
assertFlagHasAlias(t, flags, "namespace", "n")
// --now is the documented alias for --force; if it disappears,
// scripted operators that rely on it break silently.
assertFlagHasAlias(t, flags, "force", "now")

graceFlag, ok := flags["grace"].(*cli.DurationFlag)
if !ok {
t.Fatalf("--grace should be *cli.DurationFlag, got %T", flags["grace"])
}
if graceFlag.Value != monetizeapi.DefaultDrainGracePeriod {
t.Errorf("--grace default = %v, want %v", graceFlag.Value, monetizeapi.DefaultDrainGracePeriod)
}
}

func TestSellDelete_Structure(t *testing.T) {
Expand Down
29 changes: 24 additions & 5 deletions docs/guides/monetize-inference.md
Original file line number Diff line number Diff line change
Expand Up @@ -572,15 +572,34 @@ obol sell status my-qwen --namespace llm
obol sell status
```

### Pausing
### Draining

Pause an offer without deleting it:
Stop an offer gracefully so buyers can wind down before the route disappears:

```bash
obol sell stop my-qwen --namespace llm
obol sell stop my-qwen --namespace llm # default: 1h grace
obol sell stop my-qwen --namespace llm --grace 30m # custom grace
obol sell stop my-qwen --namespace llm --force # tear down immediately
```

The CR and any ERC-8004 registration remain intact. Re-create the offer with the same name to restart.
`obol sell stop` sets `spec.drainAt` on the ServiceOffer. While the offer is
draining:

- `/skill.md` and `/.well-known/agent-registration.json` advertise the offer
with `available: false` and `drainEndsAt: <RFC3339>`, so external discovery
(and ERC-8004 reputation scorers) can react before traffic disappears.
- The HTTPRoute and x402 payment gate stay up so in-flight buyers can complete
payments.
- When the grace period elapses, the controller tears down the route and marks
`Draining=False` reason=Drained.

The ServiceOffer CR and any ERC-8004 registration remain intact. Use
`obol sell delete` to remove the offer entirely.

`--force` (alias: `--now`) skips the drain window — useful when you want the
abrupt-teardown behavior of the legacy `obol.org/paused` annotation, for
example to reclaim the path immediately. Note that abrupt teardown is a worse
reputation signal for on-chain buyers than a graceful drain.

### Cleanup

Expand Down Expand Up @@ -815,7 +834,7 @@ manifest. Do not paper over smoke-test failures with an ad hoc patch.
| `obol sell http <name> --wallet ... --chain ... --per-request ... --upstream ... --port ...` | Create a ServiceOffer and register by default |
| `obol sell list` | List all ServiceOffers |
| `obol sell status <name> -n <ns>` | Show conditions for an offer |
| `obol sell stop <name> -n <ns>` | Pause an offer without deleting it |
| `obol sell stop <name> -n <ns> [--grace 1h] [--force]` | Drain an offer (advertise wind-down via discovery, then tear down the route after the grace period). `--force`/`--now` skips the grace window. |
| `obol sell delete <name> -n <ns>` | Delete an offer and cleanup |
| `obol sell status` | Show cluster pricing and registration |
| `obol sell register --private-key-file ...` | Advanced/manual registration or repair path |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,24 @@ spec:
type: string
description: "URL path prefix for the HTTPRoute, defaults to /services/<name>."
pattern: "^/[a-zA-Z0-9/_.-]*$"
drainAt:
type: string
format: date-time
description: >-
When set, marks the offer as draining. Discovery surfaces
(/skill.md and /.well-known/agent-registration.json) advertise
the offer with available=false and drainEndsAt set, so external
observers can react before the route is removed. The HTTPRoute
and payment gate stay up until drainAt + drainGracePeriod so
in-flight buyers can settle. Set by `obol sell stop`.
drainGracePeriod:
type: string
description: >-
How long after drainAt the HTTPRoute remains up. Go duration
format (e.g. "1h", "30m", "0s"). Defaults to "1h" when unset.
A zero duration tears the route down on the next reconcile
(the `obol sell stop --force` path).
pattern: "^([0-9]+(ns|us|µs|ms|s|m|h))+$"
registration:
type: object
description: >-
Expand Down Expand Up @@ -312,7 +330,9 @@ spec:
type: array
description: >-
Condition types: ModelReady, UpstreamHealthy, PaymentGateReady,
RoutePublished, Registered, Ready.
RoutePublished, Registered, Ready, Draining. Draining is True
while spec.drainAt is set and the grace window has not elapsed;
transitions to False reason=Drained once the route is torn down.
items:
type: object
required:
Expand Down
8 changes: 7 additions & 1 deletion internal/embed/skills/sell/references/serviceoffer-spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,13 @@ Each condition contains:

## Lifecycle Notes

- Pausing is represented via the `obol.org/paused: "true"` annotation.
- Graceful stop is represented via `spec.drainAt` (RFC3339 timestamp) and
the optional `spec.drainGracePeriod` (Go duration, e.g. `"30m"`, defaults
to `1h`). While draining, discovery surfaces advertise the offer with
`available: false` + `drainEndsAt`, and the HTTPRoute/payment gate stay
up until the grace period expires so in-flight buyers can settle.
`obol sell stop --force` is the equivalent of `drainGracePeriod: 0s` —
abrupt teardown with no advertised wind-down.
- Deleting a `ServiceOffer` cascades owned `Middleware` and `HTTPRoute`
resources via `ownerReferences`.
- Registration side effects are isolated in a child `RegistrationRequest`
Expand Down
113 changes: 113 additions & 0 deletions internal/monetizeapi/drain_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package monetizeapi

import (
"testing"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestServiceOffer_IsDraining(t *testing.T) {
t.Run("nil drainAt", func(t *testing.T) {
o := &ServiceOffer{}
if o.IsDraining() {
t.Errorf("IsDraining() = true, want false for nil drainAt")
}
})
t.Run("set drainAt", func(t *testing.T) {
now := metav1.Now()
o := &ServiceOffer{Spec: ServiceOfferSpec{DrainAt: &now}}
if !o.IsDraining() {
t.Errorf("IsDraining() = false, want true for non-nil drainAt")
}
})
}

func TestServiceOffer_DrainEndsAt(t *testing.T) {
base := time.Date(2026, time.May, 1, 12, 0, 0, 0, time.UTC)
baseMeta := metav1.NewTime(base)

cases := []struct {
name string
drain *metav1.Time
grace *metav1.Duration
want time.Time
}{
{
name: "nil drainAt returns zero",
drain: nil,
grace: nil,
want: time.Time{},
},
{
name: "nil grace applies default 1h",
drain: &baseMeta,
grace: nil,
want: base.Add(time.Hour),
},
{
name: "explicit zero grace honored",
drain: &baseMeta,
grace: &metav1.Duration{Duration: 0},
want: base,
},
{
name: "custom grace honored",
drain: &baseMeta,
grace: &metav1.Duration{Duration: 30 * time.Minute},
want: base.Add(30 * time.Minute),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
o := &ServiceOffer{Spec: ServiceOfferSpec{DrainAt: tc.drain, DrainGracePeriod: tc.grace}}
if got := o.DrainEndsAt(); !got.Equal(tc.want) {
t.Errorf("DrainEndsAt() = %v, want %v", got, tc.want)
}
})
}
}

func TestServiceOffer_DrainExpired(t *testing.T) {
now := time.Date(2026, time.May, 1, 12, 0, 0, 0, time.UTC)

t.Run("not draining returns false", func(t *testing.T) {
o := &ServiceOffer{}
if o.DrainExpired(now) {
t.Errorf("DrainExpired() = true, want false for non-draining offer")
}
})

t.Run("mid-drain returns false", func(t *testing.T) {
drainAt := metav1.NewTime(now.Add(-10 * time.Minute))
o := &ServiceOffer{Spec: ServiceOfferSpec{
DrainAt: &drainAt,
DrainGracePeriod: &metav1.Duration{Duration: time.Hour},
}}
if o.DrainExpired(now) {
t.Errorf("DrainExpired() = true, want false for mid-drain offer")
}
})

t.Run("expired returns true", func(t *testing.T) {
drainAt := metav1.NewTime(now.Add(-2 * time.Hour))
o := &ServiceOffer{Spec: ServiceOfferSpec{
DrainAt: &drainAt,
DrainGracePeriod: &metav1.Duration{Duration: time.Hour},
}}
if !o.DrainExpired(now) {
t.Errorf("DrainExpired() = false, want true for expired drain")
}
})

t.Run("force path zero grace tears down on next reconcile", func(t *testing.T) {
drainAt := metav1.NewTime(now)
o := &ServiceOffer{Spec: ServiceOfferSpec{
DrainAt: &drainAt,
DrainGracePeriod: &metav1.Duration{Duration: 0},
}}
if !o.DrainExpired(now) {
t.Errorf("DrainExpired() = false at now == drainAt with zero grace, want true")
}
})
}
Loading
Loading