diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 81f46aa9b0..99a11d8ae8 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,7 +10,6 @@ ### Bundles * Validate that resource keys do not contain variable references ([#5169](https://github.com/databricks/cli/pull/5169)) * engine/direct: Drop the deployment state entry on a recreate before the follow-up `Create`, so a `Create` failure no longer leaves a broken state with `invalid state: empty id` on the next `bundle plan` ([#5173](https://github.com/databricks/cli/pull/5173)). -* engine/direct: vector search endpoints: trigger recreate when endpoint is recreated out of band ([#5127](https://github.com/databricks/cli/pull/5127)) ### Dependency updates diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 5a55ba006e..c79b0d3533 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3042,7 +3042,7 @@ resources.vector_search_endpoints.*.endpoint_status *vectorsearch.EndpointStatus resources.vector_search_endpoints.*.endpoint_status.message string REMOTE resources.vector_search_endpoints.*.endpoint_status.state vectorsearch.EndpointStatusState REMOTE resources.vector_search_endpoints.*.endpoint_type vectorsearch.EndpointType ALL -resources.vector_search_endpoints.*.endpoint_uuid string REMOTE STATE +resources.vector_search_endpoints.*.endpoint_uuid string REMOTE resources.vector_search_endpoints.*.id string INPUT REMOTE resources.vector_search_endpoints.*.last_updated_timestamp int64 REMOTE resources.vector_search_endpoints.*.last_updated_user string REMOTE diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.plan.direct.json b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.plan.direct.json index dbd1364b12..93aa4f1a24 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.plan.direct.json +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.plan.direct.json @@ -3,12 +3,6 @@ "resources.vector_search_endpoints.my_endpoint": { "action": "update", "changes": { - "endpoint_uuid": { - "action": "skip", - "reason": "custom", - "old": "[MY_ENDPOINT_UUID]", - "remote": "[MY_ENDPOINT_UUID]" - }, "min_qps": { "action": "update", "old": 1, diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt index 12eedfcf38..5a5f6d22f0 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt @@ -15,7 +15,7 @@ Deployment complete! "state": "ONLINE" }, "endpoint_type": "STANDARD", - "id": "[MY_ENDPOINT_UUID]", + "id": "[UUID]", "last_updated_timestamp": [UNIX_TIME_MILLIS][1], "last_updated_user": "[USERNAME]", "name": "vs-endpoint-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script index 3c2062e474..81e86fefcb 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script @@ -11,11 +11,6 @@ trace $CLI bundle deploy endpoint_name="vs-endpoint-${UNIQUE_NAME}" -# Register a stable label for the endpoint UUID so the plan output shows the -# same token for both saved (old) and remote, confirming they match. -endpoint_uuid=$($CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq -r '.id') -add_repl.py "$endpoint_uuid" "MY_ENDPOINT_UUID" - title "Simulate remote drift: change min_qps to 5 outside the bundle" trace $CLI vector-search-endpoints patch-endpoint "${endpoint_name}" --min-qps 5 diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt index d24c6bea8d..dece842119 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt @@ -13,9 +13,6 @@ Deployment complete! "endpoint_type": "STANDARD" } ->>> print_state.py -"/vector-search-endpoints/[ORIGINAL_ENDPOINT_UUID]" - === Delete and recreate remotely with the same name >>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] @@ -35,14 +32,12 @@ Deployment complete! Original endpoint UUID: [ORIGINAL_ENDPOINT_UUID] Remote recreated endpoint UUID: [REMOTE_RECREATED_ENDPOINT_UUID] -=== Plan detects the UUID change and proposes recreate +=== Plan after out-of-band recreate >>> [CLI] bundle plan -recreate vector_search_endpoints.my_endpoint create vector_search_endpoints.my_endpoint.permissions -Plan: 2 to add, 0 to change, 1 to delete, 0 unchanged +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged -=== Deploy recreates the endpoint and rebinds permissions to the new UUID >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-recreated-same-name-[UNIQUE_NAME]/default/files... Deploying resources... @@ -51,12 +46,14 @@ Deployment complete! >>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME] { + "id": "[REMOTE_RECREATED_ENDPOINT_UUID]", "name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_type": "STANDARD" } ->>> print_state.py -"/vector-search-endpoints/[UUID]" +=== Verify no permanent drift after deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script index 8f28189d7f..dbef9250f2 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script @@ -14,7 +14,6 @@ trace $CLI bundle deploy original_endpoint_uuid=$($CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq -r '.id') add_repl.py "$original_endpoint_uuid" "ORIGINAL_ENDPOINT_UUID" trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{id, name, endpoint_type}' -trace print_state.py | jq '.state."resources.vector_search_endpoints.my_endpoint.permissions".state.object_id' title "Delete and recreate remotely with the same name" trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" @@ -32,10 +31,11 @@ if [ "$original_endpoint_uuid" = "$remote_recreated_endpoint_uuid" ]; then exit 1 fi -title "Plan detects the UUID change and proposes recreate" -trace $CLI bundle plan | contains.py "recreate vector_search_endpoints.my_endpoint" "create vector_search_endpoints.my_endpoint.permissions" +title "Plan after out-of-band recreate" +trace $CLI bundle plan -title "Deploy recreates the endpoint and rebinds permissions to the new UUID" trace $CLI bundle deploy -trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' -trace print_state.py | jq '.state."resources.vector_search_endpoints.my_endpoint.permissions".state.object_id' +trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{id, name, endpoint_type}' + +title "Verify no permanent drift after deploy" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index 1322f474a1..24bbd1a6e7 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -5,11 +5,9 @@ import ( "time" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) @@ -18,23 +16,6 @@ var ( pathMinQps = structpath.MustParsePath("min_qps") ) -// VectorSearchEndpointState is persisted in deployment state. endpoint_uuid is -// tracked so out-of-band replacement of an endpoint with the same name can be -// detected: when saved UUID differs from remote UUID, the endpoint is recreated. -type VectorSearchEndpointState struct { - vectorsearch.CreateEndpoint - EndpointUuid string `json:"endpoint_uuid,omitempty"` -} - -// Custom marshalers required because embedded CreateEndpoint has its own. -func (s *VectorSearchEndpointState) UnmarshalJSON(b []byte) error { - return marshal.Unmarshal(b, s) -} - -func (s VectorSearchEndpointState) MarshalJSON() ([]byte, error) { - return marshal.Marshal(s) -} - // VectorSearchEndpointRemote is remote state for a vector search endpoint. It embeds API response // fields for drift comparison and adds endpoint_uuid for permissions; deployment state id remains the endpoint name. type VectorSearchEndpointRemote struct { @@ -60,28 +41,22 @@ func (*ResourceVectorSearchEndpoint) New(client *databricks.WorkspaceClient) *Re return &ResourceVectorSearchEndpoint{client: client} } -func (*ResourceVectorSearchEndpoint) PrepareState(input *resources.VectorSearchEndpoint) *VectorSearchEndpointState { - return &VectorSearchEndpointState{ - CreateEndpoint: input.CreateEndpoint, - EndpointUuid: "", - } +func (*ResourceVectorSearchEndpoint) PrepareState(input *resources.VectorSearchEndpoint) *vectorsearch.CreateEndpoint { + return &input.CreateEndpoint } -func (*ResourceVectorSearchEndpoint) RemapState(remote *VectorSearchEndpointRemote) *VectorSearchEndpointState { +func (*ResourceVectorSearchEndpoint) RemapState(remote *VectorSearchEndpointRemote) *vectorsearch.CreateEndpoint { var minQps int64 if remote.ScalingInfo != nil { minQps = remote.ScalingInfo.RequestedMinQps } - return &VectorSearchEndpointState{ - CreateEndpoint: vectorsearch.CreateEndpoint{ - Name: remote.Name, - EndpointType: remote.EndpointType, - BudgetPolicyId: remote.BudgetPolicyId, - UsagePolicyId: "", // Missing in remote - MinQps: minQps, - ForceSendFields: utils.FilterFields[vectorsearch.CreateEndpoint](remote.ForceSendFields, "UsagePolicyId"), - }, - EndpointUuid: remote.EndpointUuid, + return &vectorsearch.CreateEndpoint{ + Name: remote.Name, + EndpointType: remote.EndpointType, + BudgetPolicyId: remote.BudgetPolicyId, + UsagePolicyId: "", // Missing in remote + MinQps: minQps, + ForceSendFields: utils.FilterFields[vectorsearch.CreateEndpoint](remote.ForceSendFields, "UsagePolicyId"), } } @@ -93,19 +68,16 @@ func (r *ResourceVectorSearchEndpoint) DoRead(ctx context.Context, id string) (* return newVectorSearchEndpointRemote(info), nil } -func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *VectorSearchEndpointState) (string, *VectorSearchEndpointRemote, error) { - waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, config.CreateEndpoint) +func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (string, *VectorSearchEndpointRemote, error) { + waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, *config) if err != nil { return "", nil, err } id := config.Name - if waiter.Response != nil { - config.EndpointUuid = waiter.Response.Id - } return id, newVectorSearchEndpointRemote(waiter.Response), nil } -func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, config *VectorSearchEndpointState) (*VectorSearchEndpointRemote, error) { +func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (*VectorSearchEndpointRemote, error) { info, err := r.client.VectorSearchEndpoints.WaitGetEndpointVectorSearchEndpointOnline(ctx, config.Name, 60*time.Minute, nil) if err != nil { return nil, err @@ -113,7 +85,7 @@ func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, conf return newVectorSearchEndpointRemote(info), nil } -func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string, config *VectorSearchEndpointState, entry *PlanEntry) (*VectorSearchEndpointRemote, error) { +func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string, config *vectorsearch.CreateEndpoint, entry *PlanEntry) (*VectorSearchEndpointRemote, error) { if entry.Changes.HasChange(pathBudgetPolicyId) { _, err := r.client.VectorSearchEndpoints.UpdateEndpointBudgetPolicy(ctx, vectorsearch.PatchEndpointBudgetPolicyRequest{ EndpointName: id, @@ -135,36 +107,9 @@ func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string, } } - // Preserve endpoint_uuid in saved state: PrepareState leaves it empty because - // it isn't in config, so copy from remote before SaveState writes newState. - if remote, ok := entry.RemoteState.(*VectorSearchEndpointRemote); ok && remote != nil { - config.EndpointUuid = remote.EndpointUuid - } - return nil, nil } func (r *ResourceVectorSearchEndpoint) DoDelete(ctx context.Context, id string) error { return r.client.VectorSearchEndpoints.DeleteEndpointByEndpointName(ctx, id) } - -// OverrideChangeDesc classifies endpoint_uuid drift: Recreate when saved UUID -// differs from remote (endpoint replaced out-of-band), Skip otherwise. The -// field is not in config, so a synthetic diff between saved state and an empty -// newState is expected on every plan. -func (*ResourceVectorSearchEndpoint) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *VectorSearchEndpointRemote) error { - if path.String() != "endpoint_uuid" { - return nil - } - savedUuid, _ := change.Old.(string) - var remoteUuid string - if remote != nil { - remoteUuid = remote.EndpointUuid - } - if savedUuid != "" && remoteUuid != "" && savedUuid != remoteUuid { - change.Action = deployplan.Recreate - } else { - change.Action = deployplan.Skip - } - return nil -}