From f31555cf94986dbc30ecd056038f345da3404ee7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 12 Jun 2026 11:03:59 +0000 Subject: [PATCH 01/15] bundle: expose computed volume_path during initialize Add a computed, read-only volume_path field to volume resources, set to the Unity Catalog path /Volumes/{catalog}/{schema}/{name} during the initialize phase. This enables other resources to reference ${resources.volumes..volume_path}. - InitializeVolumePaths resolves catalog_name/schema_name/name references locally to compute the path without mutating the original references, so validate and plan keep showing the references. - volume_path is only set when catalog, schema, and name resolve to concrete values; unresolved or malformed references leave it empty. - Computation runs after PythonMutator: volume_path is a computed field the PyDABs Volume model does not declare, so it must not be exposed to Python. --- .../bundle/python/volumes-support/output.txt | 2 + acceptance/bundle/refschema/out.fields.txt | 1 + .../resource_deps/bad_syntax/output.txt | 1 + .../implicit_deps_volume/output.txt | 1 + .../non_existent_field/output.txt | 1 + .../out.deploy.requests.direct.json | 6 +- .../out.deploy.requests.terraform.json | 16 +- .../remote_field_storage_location/output.txt | 2 + .../remote_field_storage_location/script | 2 +- .../databricks.yml.tmpl | 20 +++ .../remote_field_volume_path/out.test.toml | 3 + .../remote_field_volume_path/output.txt | 36 +++++ .../remote_field_volume_path/script | 4 + .../resource_deps/resources_var/output.txt | 2 + .../output.txt | 1 + .../volumes/change-comment/output.txt | 3 + .../resources/volumes/change-name/output.txt | 1 + .../volumes/change-schema-name/output.txt | 1 + .../validate/presets_name_prefix/output.txt | 3 + .../out.with_empty_prefix.json | 1 + .../out.with_nil_prefix.json | 1 + .../out.with_static_prefix.json | 1 + .../out.without_presets.json | 1 + .../config/mutator/initialize_volume_paths.go | 80 ++++++++++ .../mutator/initialize_volume_paths_test.go | 138 ++++++++++++++++++ .../mutator/resolve_variable_references.go | 25 ++++ .../resolve_variable_references_test.go | 47 ++++++ bundle/config/resources/volume.go | 21 +++ bundle/config/resources/volume_test.go | 33 +++++ bundle/configsync/defaults.go | 1 + .../deploy/terraform/tfdyn/convert_volume.go | 6 + .../terraform/tfdyn/convert_volume_test.go | 1 + bundle/phases/initialize.go | 7 + bundle/terraform_dabs_map/generated.go | 4 - 34 files changed, 457 insertions(+), 16 deletions(-) create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/output.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/script create mode 100644 bundle/config/mutator/initialize_volume_paths.go create mode 100644 bundle/config/mutator/initialize_volume_paths_test.go diff --git a/acceptance/bundle/python/volumes-support/output.txt b/acceptance/bundle/python/volumes-support/output.txt index e2b6d8771e6..eafa1e8685d 100644 --- a/acceptance/bundle/python/volumes-support/output.txt +++ b/acceptance/bundle/python/volumes-support/output.txt @@ -17,12 +17,14 @@ "catalog_name": "my_catalog", "name": "My Volume (updated)", "schema_name": "my_schema", + "volume_path": "/Volumes/my_catalog/my_schema/My Volume (updated)", "volume_type": "MANAGED" }, "my_volume_2": { "catalog_name": "my_catalog_2", "name": "My Volume (2) (updated)", "schema_name": "my_schema_2", + "volume_path": "/Volumes/my_catalog_2/my_schema_2/My Volume (2) (updated)", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index a6776c50393..11d50a4c219 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3476,6 +3476,7 @@ resources.volumes.*.updated_at int64 REMOTE resources.volumes.*.updated_by string REMOTE resources.volumes.*.url string INPUT resources.volumes.*.volume_id string REMOTE +resources.volumes.*.volume_path string INPUT resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL resources.volumes.*.grants.securable_type string ALL diff --git a/acceptance/bundle/resource_deps/bad_syntax/output.txt b/acceptance/bundle/resource_deps/bad_syntax/output.txt index 0e9d83b6431..d5bde99d707 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/output.txt +++ b/acceptance/bundle/resource_deps/bad_syntax/output.txt @@ -6,6 +6,7 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { diff --git a/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt b/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt index 2e454e81522..4258182b523 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt +++ b/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt @@ -17,6 +17,7 @@ "catalog_name": "${resources.catalogs.my_catalog.name}", "name": "myvolume", "schema_name": "${resources.schemas.my_schema.name}", + "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/non_existent_field/output.txt b/acceptance/bundle/resource_deps/non_existent_field/output.txt index ec7de848680..d7f882960b1 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/output.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/output.txt @@ -6,6 +6,7 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json index 5ce661fb129..428a0f4bc02 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json @@ -11,7 +11,8 @@ "path": "/api/2.1/unity-catalog/volumes", "body": { "catalog_name": "main", - "name": "volumebar-[UNIQUE_NAME]", + "comment": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", + "name": "volumefoo-[UNIQUE_NAME]", "schema_name": "myschema-[UNIQUE_NAME]", "volume_type": "MANAGED" } @@ -21,8 +22,7 @@ "path": "/api/2.1/unity-catalog/volumes", "body": { "catalog_name": "main", - "comment": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", - "name": "volumefoo-[UNIQUE_NAME]", + "name": "volumebar-[UNIQUE_NAME]", "schema_name": "myschema-[UNIQUE_NAME]", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json index d048e68d7be..d9976b9d099 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json @@ -1,3 +1,11 @@ +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/schemas/main.myschema-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]" +} { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", @@ -6,10 +14,6 @@ "name": "myschema-[UNIQUE_NAME]" } } -{ - "method": "GET", - "path": "/api/2.1/unity-catalog/schemas/main.myschema-[UNIQUE_NAME]" -} { "method": "POST", "path": "/api/2.1/unity-catalog/volumes", @@ -20,7 +24,3 @@ "volume_type": "MANAGED" } } -{ - "method": "GET", - "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]" -} diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt b/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt index ce5fcfb66ba..51bd4b30bc5 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt @@ -12,6 +12,7 @@ "catalog_name": "main", "name": "volumebar-[UNIQUE_NAME]", "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", "volume_type": "MANAGED" }, "foo": { @@ -19,6 +20,7 @@ "comment": "${resources.volumes.bar.storage_location}", "name": "volumefoo-[UNIQUE_NAME]", "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/script b/acceptance/bundle/resource_deps/remote_field_storage_location/script index 30ed44956f0..7c70c481882 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/script +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/script @@ -9,7 +9,7 @@ trace $CLI bundle plan trace print_requests.py --get //unity trap cleanup EXIT trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py --sort --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json # Terraform could not deploy, so it still shows up here; direct shows no drift: $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl b/acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl new file mode 100644 index 00000000000..ec1e2619848 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: testbundle-${UNIQUE_NAME} + +resources: + schemas: + my: + catalog_name: main + name: myschema-${UNIQUE_NAME} + + volumes: + bar: + catalog_name: main + schema_name: myschema-${UNIQUE_NAME} + name: volumebar-${UNIQUE_NAME} + + foo: + catalog_name: main + schema_name: myschema-${UNIQUE_NAME} + name: volumefoo-${UNIQUE_NAME} + comment: ${resources.volumes.bar.volume_path} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml b/acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/output.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/output.txt new file mode 100644 index 00000000000..2db063aca34 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/output.txt @@ -0,0 +1,36 @@ + +>>> [CLI] bundle validate -o json +{ + "schemas": { + "my": { + "catalog_name": "main", + "name": "myschema-[UNIQUE_NAME]" + } + }, + "volumes": { + "bar": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", + "volume_type": "MANAGED" + }, + "foo": { + "catalog_name": "main", + "comment": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + } +} + +>>> [CLI] bundle plan +create schemas.my +create volumes.bar +create volumes.foo + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged + +>>> print_requests.py --get //unity diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/script b/acceptance/bundle/resource_deps/remote_field_volume_path/script new file mode 100644 index 00000000000..c6c051b389b --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/script @@ -0,0 +1,4 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle plan +trace print_requests.py --get //unity diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index eb05259f812..7a52aebacb8 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -21,12 +21,14 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { "catalog_name": "${resources.volumes.bar.catalog_name}", "name": "myname", "schema_name": "${resources.volumes.bar.schema_name}", + "volume_path": "/Volumes/mycatalog/myschema/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt index 87533e080e7..08dbf899f70 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt @@ -27,6 +27,7 @@ "catalog_name": "${resources.schemas.bar.catalog_name}", "name": "myname", "schema_name": "${resources.schemas.bar.name}", + "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resources/volumes/change-comment/output.txt b/acceptance/bundle/resources/volumes/change-comment/output.txt index fa291c749f7..0bdc340ab13 100644 --- a/acceptance/bundle/resources/volumes/change-comment/output.txt +++ b/acceptance/bundle/resources/volumes/change-comment/output.txt @@ -7,6 +7,7 @@ "modified_status": "created", "name": "myvolume", "schema_name": "myschema", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } @@ -65,6 +66,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } @@ -116,5 +118,6 @@ Error: Resource catalog.VolumeInfo not found: main.myschema.myvolume "modified_status": "created", "name": "myvolume", "schema_name": "myschema", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/resources/volumes/change-name/output.txt b/acceptance/bundle/resources/volumes/change-name/output.txt index 0572daef298..db2e6493c2e 100644 --- a/acceptance/bundle/resources/volumes/change-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-name/output.txt @@ -28,6 +28,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resources/volumes/change-schema-name/output.txt b/acceptance/bundle/resources/volumes/change-schema-name/output.txt index 66292ea6621..27ca4ae1262 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-schema-name/output.txt @@ -28,6 +28,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index 08579b5dd2b..f7d3e8d2053 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -49,6 +49,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } @@ -104,6 +105,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } @@ -159,6 +161,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json index cf1745cc2f0..258b6b938ab 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json index cf1745cc2f0..258b6b938ab 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json index eff500341a0..3fe9aeea34a 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json @@ -57,6 +57,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json index cf1745cc2f0..258b6b938ab 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go new file mode 100644 index 00000000000..f1d97360a3c --- /dev/null +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -0,0 +1,80 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +type initializeVolumePaths struct{} + +// InitializeVolumePaths sets resources.volumes.*.volume_path from catalog, schema, and name. +// +// catalog_name and schema_name may be ${resources.catalogs/schemas..name} references +// (for example, after CaptureUCDependencies rewrites implicit dependencies). We resolve those +// references locally to compute the path, but we do not write the resolved values back: the +// original references are preserved so validate and plan keep showing them. +// +// The path is only set when catalog, schema, and name resolve to concrete values (no remaining +// ${...} references). This enables ${resources.volumes..volume_path} interpolation during +// initialize. +func InitializeVolumePaths() bundle.Mutator { + return &initializeVolumePaths{} +} + +func (m *initializeVolumePaths) Name() string { + return "InitializeVolumePaths" +} + +func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("volumes"), dyn.AnyKey()) + return dyn.MapByPattern(root, pattern, func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { + var vol resources.Volume + if err := convert.ToTyped(&vol, v); err != nil { + return dyn.InvalidValue, err + } + + // Resolve references for the purpose of computing the path only; the + // original field values in v are left untouched. + vol.CatalogName = resolveResourceReference(root, vol.CatalogName) + vol.SchemaName = resolveResourceReference(root, vol.SchemaName) + vol.Name = resolveResourceReference(root, vol.Name) + + path := vol.ComputeVolumePath() + if path == "" { + return v, nil + } + return dyn.Set(v, "volume_path", dyn.V(path)) + }) + }) + if err != nil { + return diag.FromErr(err) + } + return nil +} + +// resolveResourceReference returns the concrete value of a pure ${resources....} reference +// by looking it up in root. Non-reference values and references that cannot be resolved (or +// resolve to another reference) are returned unchanged, so they still contain "${" and the +// caller will not compute a volume_path for them. +func resolveResourceReference(root dyn.Value, s string) string { + p, ok := dynvar.PureReferenceToPath(s) + if !ok || p[0].Key() != "resources" { + return s + } + rv, err := dyn.GetByPath(root, p) + if err != nil { + return s + } + rs, ok := rv.AsString() + if !ok { + return s + } + return rs +} diff --git a/bundle/config/mutator/initialize_volume_paths_test.go b/bundle/config/mutator/initialize_volume_paths_test.go new file mode 100644 index 00000000000..935a0b0600d --- /dev/null +++ b/bundle/config/mutator/initialize_volume_paths_test.go @@ -0,0 +1,138 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/require" +) + +func TestInitializeVolumePaths(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Schemas: map[string]*resources.Schema{ + "my": { + CreateSchema: catalog.CreateSchema{ + CatalogName: "main", + Name: "myschema", + }, + }, + }, + Volumes: map[string]*resources.Volume{ + "bar": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volbar", + }, + }, + // foo references the schema's name; InitializeVolumePaths resolves + // it locally to compute the path without rewriting schema_name. + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.my.name}", + Name: "volfoo", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.NoError(t, diags.Error()) + + require.Equal(t, "/Volumes/main/myschema/volbar", b.Config.Resources.Volumes["bar"].VolumePath) + + foo := b.Config.Resources.Volumes["foo"] + require.Equal(t, "/Volumes/main/myschema/volfoo", foo.VolumePath) + // The schema_name reference must be preserved, not replaced with the resolved value. + require.Equal(t, "${resources.schemas.my.name}", foo.SchemaName) +} + +func TestInitializeVolumePaths_UnresolvedReference(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + // The referenced schema does not exist, so the path is left unset. + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.missing.name}", + Name: "volfoo", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.NoError(t, diags.Error()) + require.Empty(t, b.Config.Resources.Volumes["foo"].VolumePath) +} + +func TestInitializeVolumePaths_MalformedReference(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + // A malformed reference must not leak into the computed path. + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "${resources.volumes.bar.bad..syntax}", + SchemaName: "myschema", + Name: "volfoo", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.NoError(t, diags.Error()) + require.Empty(t, b.Config.Resources.Volumes["foo"].VolumePath) +} + +func TestVolumePathPipeline_ResolvesCrossVolumeReference(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "bar": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volbar", + }, + }, + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volfoo", + Comment: "${resources.volumes.bar.volume_path}", + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq( + t.Context(), + b, + InitializeVolumePaths(), + ResolveVolumePathReferencesOnlyResources(), + ) + require.NoError(t, diags.Error()) + require.Equal(t, "/Volumes/main/myschema/volbar", b.Config.Resources.Volumes["bar"].VolumePath) + require.Equal(t, "/Volumes/main/myschema/volbar", b.Config.Resources.Volumes["foo"].Comment) +} diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 113f0576394..b43777f6e34 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -50,6 +50,7 @@ type resolveVariableReferences struct { prefixes []string pattern dyn.Pattern lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error) + allowPathFn func(dyn.Path) bool extraRounds int // includeResources allows resolving variables in 'resources', otherwise, they are excluded. @@ -94,6 +95,17 @@ func ResolveVariableReferencesInLookup() bundle.Mutator { } } +// ResolveVolumePathReferencesOnlyResources resolves only references to resources.volumes.*.volume_path. +func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { + return &resolveVariableReferences{ + prefixes: []string{"resources"}, + lookupFn: lookup, + allowPathFn: isVolumePathReferencePath, + extraRounds: maxResolutionRounds - 1, + includeResources: true, + } +} + func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { if path.String() == "workspace.file_path" { @@ -229,6 +241,9 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn // Perform resolution only if the path starts with one of the specified prefixes. if slices.ContainsFunc(prefixes, path.HasPrefix) { + if m.allowPathFn != nil && !m.allowPathFn(path) { + return dyn.InvalidValue, dynvar.ErrSkipResolution + } value, err := m.lookupFn(normalized, path, b) hasUpdates = hasUpdates || (err == nil && value.IsValid()) return value, err @@ -308,3 +323,13 @@ func getAllKeys(root dyn.Value) ([]string, error) { return keys, nil } + +func isVolumePathReferencePath(path dyn.Path) bool { + if len(path) != 4 { + return false + } + return path[0].Key() == "resources" && + path[1].Key() == "volumes" && + path[2].Key() != "" && + path[3].Key() == "volume_path" +} diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 876980e9486..284b78846b4 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/require" ) @@ -63,3 +64,49 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { testCase.assert(t, b) } } + +func TestResolveVolumePathReferencesOnlyResources(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + VolumePath: "/Volumes/main/myschema/myvolume", + }, + "volumePathRef": {}, + "otherRef": {}, + }, + }, + }, + } + + b.Config.Resources.Volumes["volumePathRef"].Comment = "${resources.volumes.foo.volume_path}" + b.Config.Resources.Volumes["otherRef"].Comment = "${resources.volumes.foo.name}" + + diags := bundle.Apply(t.Context(), b, ResolveVolumePathReferencesOnlyResources()) + require.NoError(t, diags.Error()) + require.Equal(t, "/Volumes/main/myschema/myvolume", b.Config.Resources.Volumes["volumePathRef"].Comment) + require.Equal(t, "${resources.volumes.foo.name}", b.Config.Resources.Volumes["otherRef"].Comment) +} + +func TestResolveVolumePathReferencesOnlyResources_MissingTarget(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": {}, + }, + }, + }, + } + b.Config.Resources.Volumes["foo"].Comment = "${resources.volumes.missing.volume_path}" + + diags := bundle.Apply(t.Context(), b, ResolveVolumePathReferencesOnlyResources()) + require.ErrorContains(t, diags.Error(), "reference does not exist: ${resources.volumes.missing.volume_path}") +} + +func TestIsVolumePathReferencePath(t *testing.T) { + require.True(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.volume_path"))) + require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.name"))) + require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.jobs.foo.name"))) +} diff --git a/bundle/config/resources/volume.go b/bundle/config/resources/volume.go index 4b1faecc9c0..bb3b150bffc 100644 --- a/bundle/config/resources/volume.go +++ b/bundle/config/resources/volume.go @@ -2,7 +2,9 @@ package resources import ( "context" + "fmt" "net/url" + "strings" "github.com/databricks/databricks-sdk-go/apierr" @@ -17,6 +19,9 @@ type Volume struct { BaseResource catalog.CreateVolumeRequestContent + // VolumePath is /Volumes/{catalog}/{schema}/{name}. Populated during initialize; not user-configurable. + VolumePath string `json:"volume_path,omitempty" bundle:"readonly"` + // List of grants to apply on this volume. Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } @@ -69,3 +74,19 @@ func (v *Volume) GetURL() string { func (v *Volume) GetName() string { return v.Name } + +// ComputeVolumePath returns the Unity Catalog volume path when catalog, schema, and name are set and resolved. +func (v *Volume) ComputeVolumePath() string { + if v.CatalogName == "" || v.SchemaName == "" || v.Name == "" { + return "" + } + // Bail out if any component still contains a "${...}" reference. We check for the + // literal "${" rather than a well-formed reference so malformed references (which + // would otherwise leak verbatim into the path) are also rejected. + if strings.Contains(v.CatalogName, "${") || + strings.Contains(v.SchemaName, "${") || + strings.Contains(v.Name, "${") { + return "" + } + return fmt.Sprintf("/Volumes/%s/%s/%s", v.CatalogName, v.SchemaName, v.Name) +} diff --git a/bundle/config/resources/volume_test.go b/bundle/config/resources/volume_test.go index ef44a2e61de..7a1a090a56b 100644 --- a/bundle/config/resources/volume_test.go +++ b/bundle/config/resources/volume_test.go @@ -5,10 +5,43 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) +func TestComputeVolumePath(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "myvol", + }, + } + require.Equal(t, "/Volumes/main/myschema/myvol", v.ComputeVolumePath()) +} + +func TestComputeVolumePath_UnresolvedReference(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.my.name}", + Name: "myvol", + }, + } + require.Empty(t, v.ComputeVolumePath()) +} + +func TestComputeVolumePath_MissingField(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + Name: "myvol", + }, + } + require.Empty(t, v.ComputeVolumePath()) +} + func TestVolumeNotFound(t *testing.T) { ctx := t.Context() diff --git a/bundle/configsync/defaults.go b/bundle/configsync/defaults.go index cc56b361aff..9e080edeae0 100644 --- a/bundle/configsync/defaults.go +++ b/bundle/configsync/defaults.go @@ -100,6 +100,7 @@ var serverSideDefaults = map[string]any{ // Volume fields "resources.volumes.*.storage_location": alwaysSkip, + "resources.volumes.*.volume_path": alwaysSkip, // SQL warehouse fields "resources.sql_warehouses.*.creator_name": alwaysSkip, diff --git a/bundle/deploy/terraform/tfdyn/convert_volume.go b/bundle/deploy/terraform/tfdyn/convert_volume.go index 287ddee0c6b..e8becaea175 100644 --- a/bundle/deploy/terraform/tfdyn/convert_volume.go +++ b/bundle/deploy/terraform/tfdyn/convert_volume.go @@ -11,6 +11,12 @@ import ( ) func convertVolumeResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // volume_path is computed by the API and cannot be configured in Terraform. + vin, err := dyn.DropKeys(vin, []string{"volume_path"}) + if err != nil { + return dyn.InvalidValue, err + } + // Normalize the output value to the target schema. vout, diags := convert.Normalize(schema.ResourceVolume{}, vin) for _, diag := range diags { diff --git a/bundle/deploy/terraform/tfdyn/convert_volume_test.go b/bundle/deploy/terraform/tfdyn/convert_volume_test.go index bd8798780ad..f21b1024483 100644 --- a/bundle/deploy/terraform/tfdyn/convert_volume_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_volume_test.go @@ -22,6 +22,7 @@ func TestConvertVolume(t *testing.T) { StorageLocation: "s3://bucket/path", VolumeType: "EXTERNAL", }, + VolumePath: "/Volumes/catalog/schema/name", Grants: []catalog.PrivilegeAssignment{ { Privileges: []catalog.Privilege{catalog.PrivilegeReadVolume}, diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 80127843e83..f9f933d56f1 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -142,6 +142,13 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // After PythonMutator, mutators must not change bundle resources, or such changes are not // going to be visible in Python code. + // Compute resources.volumes.*.volume_path and resolve references to it. This must run + // after PythonMutator: volume_path is a computed, read-only field that the PyDABs Volume + // model does not declare, so exposing it to Python would fail resource loading. Like the + // "deployment" metadata below, it is intentionally not visible to Python code. + mutator.InitializeVolumePaths(), + mutator.ResolveVolumePathReferencesOnlyResources(), + // Resolve --select selectors against the materialized resources: normalize // each to its "type.name" form and validate it exists. Runs after all resource // mutations so that dynamically added resources are visible. This does not diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index 290d6889c06..dff638c5f58 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -33,7 +33,6 @@ package terraform_dabs_map // secret_scopes / databricks_secret_scope: 1 tf-only // sql_warehouses / databricks_sql_endpoint: 2 tf-only // synced_database_tables / databricks_database_synced_database_table: 5 dabs-only -// volumes / databricks_volume: 1 tf-only // TerraformToDABsFieldMap maps DABs group name → nested TF segments → DABs segment name. // Navigate using TF field name segments; DABs is the corresponding DABs name when it differs. @@ -651,9 +650,6 @@ var TerraformOnlyFields = map[string]FieldSet{ "data_source_id": {}, "no_wait": {}, }, - "volumes": { - "volume_path": {}, - }, } // DABsToTerraformRenameMap maps DABs group name → nested DABs segments → TF segment name. From 16e2e5e20cc4d903bad79b9e097fe205f713cce3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 12 Jun 2026 11:50:14 +0000 Subject: [PATCH 02/15] acc: add volume_path edge-case and cross-resource tests Cover behaviors the initial volume_path change did not exercise: - unresolved_volume_path: a volume whose name is only known at deploy cannot have volume_path computed at plan time, so a reference to it resolves to an empty string (no error). Documented as known badness. - volume_path_job_ref: the motivating use case from #4233, a job parameter referencing ${resources.volumes..volume_path}. - Unit test for resolving a reference to an unset volume_path on an existing volume (resolves to empty rather than erroring). --- .../unresolved_volume_path/databricks.yml | 28 +++++++++++++ .../unresolved_volume_path/out.test.toml | 3 ++ .../unresolved_volume_path/output.txt | 27 +++++++++++++ .../unresolved_volume_path/script | 1 + .../unresolved_volume_path/test.toml | 2 + .../volume_path_job_ref/databricks.yml.tmpl | 25 ++++++++++++ .../volume_path_job_ref/out.test.toml | 3 ++ .../volume_path_job_ref/output.txt | 39 +++++++++++++++++++ .../resource_deps/volume_path_job_ref/script | 3 ++ .../volume_path_job_ref/test.toml | 1 + .../resolve_variable_references_test.go | 21 ++++++++++ 11 files changed, 153 insertions(+) create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.test.toml create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/output.txt create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/script create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/test.toml create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/output.txt create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/script create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/test.toml diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml b/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml new file mode 100644 index 00000000000..e77f7100cac --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml @@ -0,0 +1,28 @@ +bundle: + name: test-bundle + +# volume_path is computed locally during initialize and is only available when +# catalog_name, schema_name, and name all resolve to concrete values. Here +# bar.name comes from baz.storage_location, a remote field that is only known +# after deploy, so bar.volume_path cannot be computed. A reference to it +# (foo.comment) therefore resolves to an empty string rather than erroring. +# +# Only `validate` is exercised: this behavior is decided in the initialize phase +# and is identical for both engines. A volume whose name is not known until +# deploy cannot be planned by Terraform anyway (name is a required argument), so +# plan/deploy would fail for reasons unrelated to volume_path. +resources: + volumes: + baz: + catalog_name: mycatalog + schema_name: myschema + name: bazname + bar: + catalog_name: mycatalog + schema_name: myschema + name: ${resources.volumes.baz.storage_location} + foo: + catalog_name: mycatalog + schema_name: myschema + name: fooname + comment: ${resources.volumes.bar.volume_path} diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.test.toml b/acceptance/bundle/resource_deps/unresolved_volume_path/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt b/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt new file mode 100644 index 00000000000..76020584e61 --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt @@ -0,0 +1,27 @@ + +>>> [CLI] bundle validate -o json +{ + "volumes": { + "bar": { + "catalog_name": "mycatalog", + "name": "${resources.volumes.baz.storage_location}", + "schema_name": "myschema", + "volume_type": "MANAGED" + }, + "baz": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/bazname", + "volume_type": "MANAGED" + }, + "foo": { + "catalog_name": "mycatalog", + "comment": "", + "name": "fooname", + "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/fooname", + "volume_type": "MANAGED" + } + } +} diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/script b/acceptance/bundle/resource_deps/unresolved_volume_path/script new file mode 100644 index 00000000000..873932a656d --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq .resources diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/test.toml b/acceptance/bundle/resource_deps/unresolved_volume_path/test.toml new file mode 100644 index 00000000000..aa19491bfe7 --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/test.toml @@ -0,0 +1,2 @@ +Badness = "foo.comment silently resolves to an empty string because bar.volume_path could not be computed; there is no error or warning pointing at the unresolved volume_path." +RecordRequests = false diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl new file mode 100644 index 00000000000..d9ac4031613 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl @@ -0,0 +1,25 @@ +bundle: + name: testbundle-${UNIQUE_NAME} + +# A job parameter references a bundle-managed volume's computed volume_path. +# This is the motivating use case from databricks/cli#4233: other resources can +# refer to ${resources.volumes..volume_path} instead of hardcoding the UC +# path. +resources: + schemas: + my: + catalog_name: main + name: myschema-${UNIQUE_NAME} + + volumes: + data: + catalog_name: main + schema_name: myschema-${UNIQUE_NAME} + name: data-${UNIQUE_NAME} + + jobs: + process: + name: process-${UNIQUE_NAME} + parameters: + - name: data_path + default: ${resources.volumes.data.volume_path} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt new file mode 100644 index 00000000000..a7c6a14c118 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt @@ -0,0 +1,39 @@ + +>>> [CLI] bundle validate -o json +{ + "process": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "process-[UNIQUE_NAME]", + "parameters": [ + { + "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "name": "data_path" + } + ], + "queue": { + "enabled": true + } + } +} +{ + "data": { + "catalog_name": "main", + "name": "data-[UNIQUE_NAME]", + "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +>>> [CLI] bundle plan +create jobs.process +create schemas.my +create volumes.data + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script new file mode 100644 index 00000000000..1e16c483c6f --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -0,0 +1,3 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle validate -o json | jq '.resources.jobs, .resources.volumes' +trace $CLI bundle plan diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 284b78846b4..5d9646b7f22 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -105,6 +105,27 @@ func TestResolveVolumePathReferencesOnlyResources_MissingTarget(t *testing.T) { require.ErrorContains(t, diags.Error(), "reference does not exist: ${resources.volumes.missing.volume_path}") } +func TestResolveVolumePathReferencesOnlyResources_UnsetTargetResolvesToEmpty(t *testing.T) { + // When the target volume exists but its volume_path was never computed (for + // example because its name is only known at deploy), the field is normalized + // to an empty string, so the reference resolves to "" rather than erroring. + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": {}, + "ref": {}, + }, + }, + }, + } + b.Config.Resources.Volumes["ref"].Comment = "${resources.volumes.foo.volume_path}" + + diags := bundle.Apply(t.Context(), b, ResolveVolumePathReferencesOnlyResources()) + require.NoError(t, diags.Error()) + require.Empty(t, b.Config.Resources.Volumes["ref"].Comment) +} + func TestIsVolumePathReferencePath(t *testing.T) { require.True(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.volume_path"))) require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.name"))) From 4883fe423d31cc8cb3fea50d390b6536989cc0e7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 12 Jun 2026 12:06:03 +0000 Subject: [PATCH 03/15] acc: add NEXT_CHANGELOG entry for volume_path --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 1f46a0a57c6..940d652f66f 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,6 +9,7 @@ ### Bundles * `bundle run` now prints the modern job run URL (`/jobs//runs/`) so that non-admin users permitted to view the run are taken to the run instead of the workspace homepage. * Fix missing field descriptions in the bundle JSON schema for fields whose upstream API docs arrived after the field was first annotated (e.g. `vector_search_endpoints.*.target_qps`); stale placeholder markers no longer hide them ([#5588](https://github.com/databricks/cli/pull/5588)). +* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes..volume_path}` instead of hardcoding `/Volumes///` ([#5550](https://github.com/databricks/cli/pull/5550)). ### Dependency updates * Bump `github.com/databricks/databricks-sdk-go` from v0.141.0 to v0.147.0 ([#5636](https://github.com/databricks/cli/pull/5636)). From fc1dce6bcbb3c6c5e994b9f21454f54f24808b57 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 07:37:31 +0000 Subject: [PATCH 04/15] acc: record per-engine plan and deploy requests for volume_path job ref Extend the volume_path_job_ref test to confirm ${resources.volumes.data.volume_path} is interpolated end-to-end: record the JSON plan per engine and capture the jobs/create request from a real deploy. Enable RecordRequests so the requests can be inspected. --- .../volume_path_job_ref/out.deploy.direct.txt | 6 ++ .../out.deploy.requests.direct.json | 23 ++++++++ .../out.deploy.requests.terraform.json | 23 ++++++++ .../out.deploy.terraform.txt | 6 ++ .../volume_path_job_ref/out.plan.direct.json | 58 +++++++++++++++++++ .../out.plan.terraform.json | 16 +++++ .../volume_path_job_ref/output.txt | 7 +-- .../resource_deps/volume_path_job_ref/script | 11 +++- .../volume_path_job_ref/test.toml | 2 +- 9 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.direct.txt create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json create mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.direct.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.direct.txt new file mode 100644 index 00000000000..5a4539339c5 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json new file mode 100644 index 00000000000..9599823840d --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json @@ -0,0 +1,23 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "process-[UNIQUE_NAME]", + "parameters": [ + { + "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "name": "data_path" + } + ], + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json new file mode 100644 index 00000000000..9599823840d --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json @@ -0,0 +1,23 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "process-[UNIQUE_NAME]", + "parameters": [ + { + "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "name": "data_path" + } + ], + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt new file mode 100644 index 00000000000..5a4539339c5 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json new file mode 100644 index 00000000000..bc0c26ecb1c --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json @@ -0,0 +1,58 @@ + +>>> errcode [CLI] bundle plan -o json +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.process": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "process-[UNIQUE_NAME]", + "parameters": [ + { + "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "name": "data_path" + } + ], + "queue": { + "enabled": true + } + } + } + }, + "resources.schemas.my": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myschema-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.data": { + "depends_on": [ + { + "node": "resources.schemas.my", + "label": "${resources.schemas.my.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "data-[UNIQUE_NAME]", + "schema_name": "myschema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json new file mode 100644 index 00000000000..90a3acec738 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json @@ -0,0 +1,16 @@ + +>>> errcode [CLI] bundle plan -o json +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.process": { + "action": "create" + }, + "resources.schemas.my": { + "action": "create" + }, + "resources.volumes.data": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt index a7c6a14c118..55c41e798a4 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt @@ -31,9 +31,4 @@ } } ->>> [CLI] bundle plan -create jobs.process -create schemas.my -create volumes.data - -Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged +>>> print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index 1e16c483c6f..eb1a5f6bc73 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,3 +1,12 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -o json | jq '.resources.jobs, .resources.volumes' -trace $CLI bundle plan + +# (a) The job's data_path default references the volume's computed volume_path. +# Record the JSON plan per engine to confirm the reference is resolved into the +# planned new_state rather than left as ${resources...}. +trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +# (b) Deploy and inspect the create-job request: the parameter default sent to +# the Jobs API must be the interpolated volume_path. +trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py //jobs > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml index a030353d571..159efe02696 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -1 +1 @@ -RecordRequests = false +RecordRequests = true From 5dd57cee84d303bdb9277073b887cba7b04fef3b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 08:50:26 +0000 Subject: [PATCH 05/15] bundle: embed unresolved references into computed volume_path A volume component (catalog_name, schema_name, name) that cannot be resolved locally is now embedded verbatim into volume_path as a ${...} reference instead of suppressing the path. The embedded reference is carried through ${resources.volumes..volume_path} interpolation and resolved later by the engine during plan or deploy, like any other resource reference. Because volume_path is a readonly field that is dropped from state before deploy, makePlan no longer treats a reference carried by such a field as a dependency (the reference is still made available to readers during initialize). --- .../config/mutator/initialize_volume_paths.go | 8 +++--- .../mutator/initialize_volume_paths_test.go | 7 +++-- bundle/config/resources/volume.go | 26 +++++++++++-------- bundle/config/resources/volume_test.go | 15 ++++++++++- bundle/direct/bundle_plan.go | 19 ++++++++++++++ 5 files changed, 58 insertions(+), 17 deletions(-) diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go index f1d97360a3c..b58c04fcf89 100644 --- a/bundle/config/mutator/initialize_volume_paths.go +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -20,9 +20,11 @@ type initializeVolumePaths struct{} // references locally to compute the path, but we do not write the resolved values back: the // original references are preserved so validate and plan keep showing them. // -// The path is only set when catalog, schema, and name resolve to concrete values (no remaining -// ${...} references). This enables ${resources.volumes..volume_path} interpolation during -// initialize. +// A component that cannot be resolved locally (for example a remote field only known at plan or +// deploy time) is left as a ${...} reference and embedded into volume_path. The embedded +// reference is then carried through ${resources.volumes..volume_path} interpolation and +// resolved later by the engine during plan or deploy, the same way any other resource reference +// is. This enables ${resources.volumes..volume_path} interpolation during initialize. func InitializeVolumePaths() bundle.Mutator { return &initializeVolumePaths{} } diff --git a/bundle/config/mutator/initialize_volume_paths_test.go b/bundle/config/mutator/initialize_volume_paths_test.go index 935a0b0600d..e680441a692 100644 --- a/bundle/config/mutator/initialize_volume_paths_test.go +++ b/bundle/config/mutator/initialize_volume_paths_test.go @@ -60,7 +60,8 @@ func TestInitializeVolumePaths_UnresolvedReference(t *testing.T) { Config: config.Root{ Resources: config.Resources{ Volumes: map[string]*resources.Volume{ - // The referenced schema does not exist, so the path is left unset. + // The reference cannot be resolved locally, so it is embedded into + // the path verbatim to be resolved later during plan or deploy. "foo": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ CatalogName: "main", @@ -75,7 +76,9 @@ func TestInitializeVolumePaths_UnresolvedReference(t *testing.T) { diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) require.NoError(t, diags.Error()) - require.Empty(t, b.Config.Resources.Volumes["foo"].VolumePath) + require.Equal(t, "/Volumes/main/${resources.schemas.missing.name}/volfoo", b.Config.Resources.Volumes["foo"].VolumePath) + // The schema_name reference itself must be preserved, not rewritten. + require.Equal(t, "${resources.schemas.missing.name}", b.Config.Resources.Volumes["foo"].SchemaName) } func TestInitializeVolumePaths_MalformedReference(t *testing.T) { diff --git a/bundle/config/resources/volume.go b/bundle/config/resources/volume.go index bb3b150bffc..9e97b5c3a27 100644 --- a/bundle/config/resources/volume.go +++ b/bundle/config/resources/volume.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" @@ -75,18 +76,21 @@ func (v *Volume) GetName() string { return v.Name } -// ComputeVolumePath returns the Unity Catalog volume path when catalog, schema, and name are set and resolved. +// ComputeVolumePath returns the Unity Catalog volume path /Volumes/{catalog}/{schema}/{name}. +// +// A component that is still a pure ${...} reference (for example a remote field only +// known at plan or deploy time) is embedded verbatim, so the reference is carried into +// the path and resolved later by the normal interpolation passes. A component that +// contains "${" but is not a well-formed reference (malformed or partial) is rejected +// to keep it from leaking into the path, in which case the empty string is returned. func (v *Volume) ComputeVolumePath() string { - if v.CatalogName == "" || v.SchemaName == "" || v.Name == "" { - return "" - } - // Bail out if any component still contains a "${...}" reference. We check for the - // literal "${" rather than a well-formed reference so malformed references (which - // would otherwise leak verbatim into the path) are also rejected. - if strings.Contains(v.CatalogName, "${") || - strings.Contains(v.SchemaName, "${") || - strings.Contains(v.Name, "${") { - return "" + for _, component := range []string{v.CatalogName, v.SchemaName, v.Name} { + if component == "" { + return "" + } + if strings.Contains(component, "${") && !dynvar.IsPureVariableReference(component) { + return "" + } } return fmt.Sprintf("/Volumes/%s/%s/%s", v.CatalogName, v.SchemaName, v.Name) } diff --git a/bundle/config/resources/volume_test.go b/bundle/config/resources/volume_test.go index 7a1a090a56b..cb14a19ed45 100644 --- a/bundle/config/resources/volume_test.go +++ b/bundle/config/resources/volume_test.go @@ -21,7 +21,7 @@ func TestComputeVolumePath(t *testing.T) { require.Equal(t, "/Volumes/main/myschema/myvol", v.ComputeVolumePath()) } -func TestComputeVolumePath_UnresolvedReference(t *testing.T) { +func TestComputeVolumePath_PureReferenceEmbedded(t *testing.T) { v := &Volume{ CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ CatalogName: "main", @@ -29,6 +29,19 @@ func TestComputeVolumePath_UnresolvedReference(t *testing.T) { Name: "myvol", }, } + // A pure reference is embedded verbatim so it can be resolved later (plan/deploy). + require.Equal(t, "/Volumes/main/${resources.schemas.my.name}/myvol", v.ComputeVolumePath()) +} + +func TestComputeVolumePath_MalformedReference(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.my.bad..syntax}", + Name: "myvol", + }, + } + // A malformed reference must not leak into the path. require.Empty(t, v.ComputeVolumePath()) } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index a9527ce56d5..2825aca71ca 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -940,6 +940,25 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root maps.Copy(refs, baseRefs) + // References are resolved against the resource's state type, not its input + // config (see the note above and dresources.TestInputSubset). A field that + // exists in input but not in state — most notably a bundle:"readonly" field + // such as volumes' computed volume_path — is dropped from state before + // deploy, so a reference it carries cannot be resolved into the state and + // must not be treated as a dependency here. Such references are still made + // available to other resources that read the field (for example + // ${resources.volumes.x.volume_path}) earlier during initialize. + stateType := adapter.StateType() + for fieldPath := range refs { + parsed, err := structpath.ParsePath(fieldPath) + if err != nil { + return nil, fmt.Errorf("%s: parsing reference path %q: %w", prefix, fieldPath, err) + } + if structaccess.ValidatePath(stateType, parsed) != nil { + delete(refs, fieldPath) + } + } + var dependsOn []deployplan.DependsOnEntry for _, reference := range refs { ref, ok := dynvar.NewRef(dyn.V(reference)) From 6d09441aa925c25b6e263ad34e428e49bb6d27d8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 08:50:40 +0000 Subject: [PATCH 06/15] acc: show volume_path reference resolving at deploy Rewrite unresolved_volume_path so an embedded ${...} reference in volume_path is resolved at deploy rather than treated as an error: bar.name comes from baz.id and foo.comment reads bar.volume_path, and the recorded deploy requests confirm the path is fully interpolated. Update non_existent_field output for the embedded-reference volume_path. --- .../non_existent_field/output.txt | 1 + .../unresolved_volume_path/databricks.yml | 27 +++++++----- .../out.deploy.direct.txt | 6 +++ .../out.deploy.requests.direct.json | 31 +++++++++++++ .../out.deploy.requests.terraform.json | 43 +++++++++++++++++++ .../out.deploy.terraform.txt | 6 +++ .../out.destroy.direct.txt | 17 ++++++++ .../out.destroy.terraform.txt | 17 ++++++++ .../out.destroy_requests.direct.json | 24 +++++++++++ .../out.destroy_requests.terraform.json | 24 +++++++++++ .../unresolved_volume_path/output.txt | 12 +++++- .../unresolved_volume_path/script | 14 ++++++ .../unresolved_volume_path/test.toml | 2 - .../resource_deps/volume_path_job_ref/script | 4 +- 14 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.direct.txt create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.direct.json create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.terraform.json create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.terraform.txt create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.direct.txt create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.terraform.txt create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.direct.json create mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.terraform.json delete mode 100644 acceptance/bundle/resource_deps/unresolved_volume_path/test.toml diff --git a/acceptance/bundle/resource_deps/non_existent_field/output.txt b/acceptance/bundle/resource_deps/non_existent_field/output.txt index d7f882960b1..09d3f03d6d7 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/output.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/output.txt @@ -13,6 +13,7 @@ "catalog_name": "${resources.volumes.bar.non_existent}", "name": "myname", "schema_name": "${resources.volumes.bar.schema_name}", + "volume_path": "/Volumes/${resources.volumes.bar.non_existent}/myschema/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml b/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml index e77f7100cac..df9e60f9d67 100644 --- a/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml @@ -1,16 +1,23 @@ bundle: name: test-bundle -# volume_path is computed locally during initialize and is only available when -# catalog_name, schema_name, and name all resolve to concrete values. Here -# bar.name comes from baz.storage_location, a remote field that is only known -# after deploy, so bar.volume_path cannot be computed. A reference to it -# (foo.comment) therefore resolves to an empty string rather than erroring. +# volume_path is computed locally during the initialize phase. It is a CLI-only +# field, not part of the volume's API state. When a component (catalog_name, +# schema_name, or name) is still an unresolved ${...} reference, that reference is +# embedded into volume_path instead of being dropped, so it can be resolved later +# during plan or deploy like any other reference. # -# Only `validate` is exercised: this behavior is decided in the initialize phase -# and is identical for both engines. A volume whose name is not known until -# deploy cannot be planned by Terraform anyway (name is a required argument), so -# plan/deploy would fail for reasons unrelated to volume_path. +# bar.name is ${resources.volumes.baz.id}, a remote value only known after baz is +# created. bar.volume_path is therefore computed at initialize as +# /Volumes/mycatalog/myschema/${resources.volumes.baz.id}, with the reference left +# intact. foo.comment = ${resources.volumes.bar.volume_path} picks up that string +# during initialize, and its embedded baz.id reference resolves at deploy, so foo +# is created with the fully resolved path (see the deploy requests below). +# +# Both validate and deploy are exercised. (baz.id is used rather than +# baz.storage_location only because the resolved value becomes bar's name, and the +# testserver cannot route a volume name that contains the slashes of a +# storage_location URL.) resources: volumes: baz: @@ -20,7 +27,7 @@ resources: bar: catalog_name: mycatalog schema_name: myschema - name: ${resources.volumes.baz.storage_location} + name: ${resources.volumes.baz.id} foo: catalog_name: mycatalog schema_name: myschema diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.direct.txt b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.direct.txt new file mode 100644 index 00000000000..0fd05668156 --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.direct.json new file mode 100644 index 00000000000..df6294fddfc --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.direct.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname", + "name": "fooname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "mycatalog.myschema.bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.terraform.json new file mode 100644 index 00000000000..a8cb2d0afcd --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.terraform.json @@ -0,0 +1,43 @@ +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname", + "name": "fooname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "mycatalog.myschema.bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.terraform.txt new file mode 100644 index 00000000000..0fd05668156 --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.terraform.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.direct.txt b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.direct.txt new file mode 100644 index 00000000000..a60dfafafad --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.direct.txt @@ -0,0 +1,17 @@ +The following resources will be deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.terraform.txt b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.terraform.txt new file mode 100644 index 00000000000..a60dfafafad --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.terraform.txt @@ -0,0 +1,17 @@ +The following resources will be deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.direct.json b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.direct.json new file mode 100644 index 00000000000..b7a38b0bbb4 --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.direct.json @@ -0,0 +1,24 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.terraform.json b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.terraform.json new file mode 100644 index 00000000000..b7a38b0bbb4 --- /dev/null +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.terraform.json @@ -0,0 +1,24 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt b/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt index 76020584e61..8cbd00ede13 100644 --- a/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt @@ -4,8 +4,9 @@ "volumes": { "bar": { "catalog_name": "mycatalog", - "name": "${resources.volumes.baz.storage_location}", + "name": "${resources.volumes.baz.id}", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}", "volume_type": "MANAGED" }, "baz": { @@ -17,7 +18,7 @@ }, "foo": { "catalog_name": "mycatalog", - "comment": "", + "comment": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}", "name": "fooname", "schema_name": "myschema", "volume_path": "/Volumes/mycatalog/myschema/fooname", @@ -25,3 +26,10 @@ } } } + +>>> [CLI] bundle plan +create volumes.bar +create volumes.baz +create volumes.foo + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/script b/acceptance/bundle/resource_deps/unresolved_volume_path/script index 873932a656d..5af614a9f4f 100644 --- a/acceptance/bundle/resource_deps/unresolved_volume_path/script +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/script @@ -1 +1,15 @@ +cleanup() { + $CLI bundle destroy --auto-approve &> out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt + print_requests.py --sort --get //unity &> out.destroy_requests.$DATABRICKS_BUNDLE_ENGINE.json +} + trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle plan + +trap cleanup EXIT +trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + +# bar.name is created from baz.id (resolved during deploy), and foo.comment is +# created with bar.volume_path, whose embedded baz.id reference also resolves +# during deploy: +print_requests.py --sort --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/test.toml b/acceptance/bundle/resource_deps/unresolved_volume_path/test.toml deleted file mode 100644 index aa19491bfe7..00000000000 --- a/acceptance/bundle/resource_deps/unresolved_volume_path/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Badness = "foo.comment silently resolves to an empty string because bar.volume_path could not be computed; there is no error or warning pointing at the unresolved volume_path." -RecordRequests = false diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index eb1a5f6bc73..bcc1cf937cb 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,12 +1,12 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -o json | jq '.resources.jobs, .resources.volumes' -# (a) The job's data_path default references the volume's computed volume_path. +# The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the # planned new_state rather than left as ${resources...}. trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json -# (b) Deploy and inspect the create-job request: the parameter default sent to +# Deploy and inspect the create-job request: the parameter default sent to # the Jobs API must be the interpolated volume_path. trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py //jobs > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json From 84be784c1a37909997f054e701bff6937ea0af8a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 08:59:40 +0000 Subject: [PATCH 07/15] acc: rename remote_field_volume_path test to computed_volume_path and trim comment Rename the volume_path acceptance fixture to better reflect what it covers, and shorten the explanatory comment in unresolved_volume_path. --- .../databricks.yml.tmpl | 0 .../out.test.toml | 0 .../output.txt | 0 .../script | 0 .../unresolved_volume_path/databricks.yml | 24 +++++++------------ 5 files changed, 9 insertions(+), 15 deletions(-) rename acceptance/bundle/resource_deps/{remote_field_volume_path => computed_volume_path}/databricks.yml.tmpl (100%) rename acceptance/bundle/resource_deps/{remote_field_volume_path => computed_volume_path}/out.test.toml (100%) rename acceptance/bundle/resource_deps/{remote_field_volume_path => computed_volume_path}/output.txt (100%) rename acceptance/bundle/resource_deps/{remote_field_volume_path => computed_volume_path}/script (100%) diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl b/acceptance/bundle/resource_deps/computed_volume_path/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl rename to acceptance/bundle/resource_deps/computed_volume_path/databricks.yml.tmpl diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml similarity index 100% rename from acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml rename to acceptance/bundle/resource_deps/computed_volume_path/out.test.toml diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/output.txt b/acceptance/bundle/resource_deps/computed_volume_path/output.txt similarity index 100% rename from acceptance/bundle/resource_deps/remote_field_volume_path/output.txt rename to acceptance/bundle/resource_deps/computed_volume_path/output.txt diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/script b/acceptance/bundle/resource_deps/computed_volume_path/script similarity index 100% rename from acceptance/bundle/resource_deps/remote_field_volume_path/script rename to acceptance/bundle/resource_deps/computed_volume_path/script diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml b/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml index df9e60f9d67..190e9411d01 100644 --- a/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml +++ b/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml @@ -1,23 +1,17 @@ bundle: name: test-bundle -# volume_path is computed locally during the initialize phase. It is a CLI-only -# field, not part of the volume's API state. When a component (catalog_name, -# schema_name, or name) is still an unresolved ${...} reference, that reference is -# embedded into volume_path instead of being dropped, so it can be resolved later -# during plan or deploy like any other reference. +# volume_path is a CLI-only computed field (not API state). When a component +# (catalog_name, schema_name, name) is an unresolved ${...} reference, it stays +# embedded in volume_path so it resolves later at plan/deploy. # -# bar.name is ${resources.volumes.baz.id}, a remote value only known after baz is -# created. bar.volume_path is therefore computed at initialize as -# /Volumes/mycatalog/myschema/${resources.volumes.baz.id}, with the reference left -# intact. foo.comment = ${resources.volumes.bar.volume_path} picks up that string -# during initialize, and its embedded baz.id reference resolves at deploy, so foo -# is created with the fully resolved path (see the deploy requests below). +# Chain under test: bar.name is a remote value (baz.id), so bar.volume_path is +# computed at initialize with that reference intact; foo.comment picks up that +# string and resolves at deploy to the fully-resolved path. # -# Both validate and deploy are exercised. (baz.id is used rather than -# baz.storage_location only because the resolved value becomes bar's name, and the -# testserver cannot route a volume name that contains the slashes of a -# storage_location URL.) +# baz.id is used instead of baz.storage_location because the resolved value +# becomes bar's name, and the testserver can't route a name containing the +# slashes of a storage_location URL. resources: volumes: baz: From d44f9f59e199789fc9a34c0fc6f905965e52f1b7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 17 Jun 2026 18:58:53 +0000 Subject: [PATCH 08/15] acc: drop --sort from remote_field_storage_location deploy requests The deploy order is deterministic via the dependency chain (schema -> bar -> foo, since foo.comment references bar.storage_location), so sorting only hid the ordering signal the test exists to verify. --- .../out.deploy.requests.direct.json | 6 +++--- .../out.deploy.requests.terraform.json | 16 ++++++++-------- .../remote_field_storage_location/script | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json index 428a0f4bc02..5ce661fb129 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json @@ -11,8 +11,7 @@ "path": "/api/2.1/unity-catalog/volumes", "body": { "catalog_name": "main", - "comment": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", - "name": "volumefoo-[UNIQUE_NAME]", + "name": "volumebar-[UNIQUE_NAME]", "schema_name": "myschema-[UNIQUE_NAME]", "volume_type": "MANAGED" } @@ -22,7 +21,8 @@ "path": "/api/2.1/unity-catalog/volumes", "body": { "catalog_name": "main", - "name": "volumebar-[UNIQUE_NAME]", + "comment": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", + "name": "volumefoo-[UNIQUE_NAME]", "schema_name": "myschema-[UNIQUE_NAME]", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json index d9976b9d099..d048e68d7be 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.terraform.json @@ -1,11 +1,3 @@ -{ - "method": "GET", - "path": "/api/2.1/unity-catalog/schemas/main.myschema-[UNIQUE_NAME]" -} -{ - "method": "GET", - "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]" -} { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", @@ -14,6 +6,10 @@ "name": "myschema-[UNIQUE_NAME]" } } +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/schemas/main.myschema-[UNIQUE_NAME]" +} { "method": "POST", "path": "/api/2.1/unity-catalog/volumes", @@ -24,3 +20,7 @@ "volume_type": "MANAGED" } } +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/script b/acceptance/bundle/resource_deps/remote_field_storage_location/script index 7c70c481882..30ed44956f0 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/script +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/script @@ -9,7 +9,7 @@ trace $CLI bundle plan trace print_requests.py --get //unity trap cleanup EXIT trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py --sort --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json # Terraform could not deploy, so it still shows up here; direct shows no drift: $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt From 401c22c096bec6e485e133841c9cf45c927af2f8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 08:09:56 +0000 Subject: [PATCH 09/15] Filter readonly references against state type inside extractReferences Pass the resource's state type into extractReferences and drop references to fields absent from state (e.g. volumes' computed volume_path) during the walk, instead of post-filtering the combined refs map. Validating the structpath we already build also avoids a string round-trip via structpath.ParsePath. Also rename the acceptance test unresolved_volume_path -> volume_path_contains_id and fix its script to invoke "$CLI bundle plan -o json". --- .../databricks.yml | 0 .../out.deploy.direct.txt | 0 .../out.deploy.requests.direct.json | 0 .../out.deploy.requests.terraform.json | 0 .../out.deploy.terraform.txt | 0 .../out.destroy.direct.txt | 0 .../out.destroy.terraform.txt | 0 .../out.destroy_requests.direct.json | 0 .../out.destroy_requests.terraform.json | 0 .../out.plan.direct.json | 58 +++++++++++++++++++ .../out.plan.terraform.json | 14 +++++ .../out.test.toml | 0 .../output.txt | 0 .../script | 1 + bundle/direct/bundle_plan.go | 45 +++++++------- 15 files changed, 93 insertions(+), 25 deletions(-) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/databricks.yml (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.deploy.direct.txt (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.deploy.requests.direct.json (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.deploy.requests.terraform.json (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.deploy.terraform.txt (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.destroy.direct.txt (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.destroy.terraform.txt (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.destroy_requests.direct.json (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.destroy_requests.terraform.json (100%) create mode 100644 acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json create mode 100644 acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.terraform.json rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/out.test.toml (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/output.txt (100%) rename acceptance/bundle/resource_deps/{unresolved_volume_path => volume_path_contains_id}/script (90%) diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml b/acceptance/bundle/resource_deps/volume_path_contains_id/databricks.yml similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/databricks.yml rename to acceptance/bundle/resource_deps/volume_path_contains_id/databricks.yml diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.direct.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.direct.txt similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.direct.txt rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.direct.txt diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.direct.json similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.direct.json rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.direct.json diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.terraform.json similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.requests.terraform.json rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.terraform.json diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.terraform.txt similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.deploy.terraform.txt rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.terraform.txt diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.direct.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.direct.txt similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.direct.txt rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.direct.txt diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.terraform.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.terraform.txt similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy.terraform.txt rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.terraform.txt diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.direct.json similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.direct.json rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.direct.json diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.terraform.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.terraform.json similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.destroy_requests.terraform.json rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.terraform.json diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json new file mode 100644 index 00000000000..fd53a596773 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json @@ -0,0 +1,58 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.volumes.bar": { + "depends_on": [ + { + "node": "resources.volumes.baz", + "label": "${resources.volumes.baz.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "mycatalog", + "name": "${resources.volumes.baz.id}", + "schema_name": "myschema", + "volume_type": "MANAGED" + }, + "vars": { + "name": "${resources.volumes.baz.id}" + } + } + }, + "resources.volumes.baz": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + } + }, + "resources.volumes.foo": { + "depends_on": [ + { + "node": "resources.volumes.baz", + "label": "${resources.volumes.baz.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "mycatalog", + "comment": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}", + "name": "fooname", + "schema_name": "myschema", + "volume_type": "MANAGED" + }, + "vars": { + "comment": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.terraform.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.terraform.json new file mode 100644 index 00000000000..44882021fba --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.terraform.json @@ -0,0 +1,14 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.volumes.bar": { + "action": "create" + }, + "resources.volumes.baz": { + "action": "create" + }, + "resources.volumes.foo": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/out.test.toml b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/out.test.toml rename to acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/output.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/output.txt similarity index 100% rename from acceptance/bundle/resource_deps/unresolved_volume_path/output.txt rename to acceptance/bundle/resource_deps/volume_path_contains_id/output.txt diff --git a/acceptance/bundle/resource_deps/unresolved_volume_path/script b/acceptance/bundle/resource_deps/volume_path_contains_id/script similarity index 90% rename from acceptance/bundle/resource_deps/unresolved_volume_path/script rename to acceptance/bundle/resource_deps/volume_path_contains_id/script index 5af614a9f4f..97b2c4d693e 100644 --- a/acceptance/bundle/resource_deps/unresolved_volume_path/script +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/script @@ -5,6 +5,7 @@ cleanup() { trace $CLI bundle validate -o json | jq .resources trace $CLI bundle plan +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trap cleanup EXIT trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 2825aca71ca..b949c3b0659 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -933,32 +933,13 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // This means input and state must be compatible: input can have more fields, but existing fields should not be moved // This means one cannot refer to fields not present in state (e.g. ${resources.jobs.foo.permissions}) - refs, err := extractReferences(configRoot.Value(), node) + refs, err := extractReferences(configRoot.Value(), node, adapter.StateType()) if err != nil { return nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) } maps.Copy(refs, baseRefs) - // References are resolved against the resource's state type, not its input - // config (see the note above and dresources.TestInputSubset). A field that - // exists in input but not in state — most notably a bundle:"readonly" field - // such as volumes' computed volume_path — is dropped from state before - // deploy, so a reference it carries cannot be resolved into the state and - // must not be treated as a dependency here. Such references are still made - // available to other resources that read the field (for example - // ${resources.volumes.x.volume_path}) earlier during initialize. - stateType := adapter.StateType() - for fieldPath := range refs { - parsed, err := structpath.ParsePath(fieldPath) - if err != nil { - return nil, fmt.Errorf("%s: parsing reference path %q: %w", prefix, fieldPath, err) - } - if structaccess.ValidatePath(stateType, parsed) != nil { - delete(refs, fieldPath) - } - } - var dependsOn []deployplan.DependsOnEntry for _, reference := range refs { ref, ok := dynvar.NewRef(dyn.V(reference)) @@ -1036,7 +1017,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } -func extractReferences(root dyn.Value, node string) (map[string]string, error) { +func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { nodeType := config.GetResourceTypeFromKey(node) refs := make(map[string]string) @@ -1064,11 +1045,25 @@ func extractReferences(root dyn.Value, node string) (map[string]string, error) { if !ok { return nil } + // Convert dyn.Path to structpath: dyn.Path.String() uses dot notation + // which is ambiguous for keys containing dots; structpath uses bracket + // notation (['key.with.dots']) which round-trips correctly. + fieldPath := dynPathToStructPath(p) + + // References are resolved against the resource's state type, not its input + // config (see the note in PlanResources and dresources.TestInputSubset). A + // field that exists in input but not in state — most notably a + // bundle:"readonly" field such as volumes' computed volume_path — is dropped + // from state before deploy, so a reference it carries cannot be resolved into + // the state and must not be treated as a dependency here. Such references are + // still made available to other resources that read the field (for example + // ${resources.volumes.x.volume_path}) earlier during initialize. + if structaccess.ValidatePath(stateType, fieldPath) != nil { + return nil + } + // Store the original string that contains references, not individual references. - // Convert dyn.Path to structpath string because refs are later parsed by structpath.ParsePath. - // dyn.Path.String() uses dot notation which is ambiguous for keys containing dots; - // structpath uses bracket notation (['key.with.dots']) which round-trips correctly. - refs[dynPathToStructPath(p).String()] = ref.Str + refs[fieldPath.String()] = ref.Str return nil }) if err != nil { From c91c9e232d32eb20fe0dae7408ba02f93fc95229 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 08:37:36 +0000 Subject: [PATCH 10/15] acc: exercise volume_path with embedded remote reference Extend volume_path_job_ref so the volume's schema_name/name reference another job's remote creator_user_name, so volume_path embeds an unresolved reference at initialize that resolves at deploy. Add the same scenario as a no_drift invariant config. --- .../configs/volume_path_job_ref.yml.tmpl | 24 +++++++++ .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + .../volume_path_job_ref/databricks.yml.tmpl | 25 +++++---- .../out.deploy.requests.direct.json | 19 ++++++- .../out.deploy.requests.terraform.json | 23 --------- .../out.deploy.terraform.txt | 40 +++++++++++++-- .../volume_path_job_ref/out.plan.direct.json | 50 +++++++++++++----- .../out.plan.terraform.json | 51 ++++++++++++++----- .../volume_path_job_ref/output.txt | 21 ++++++-- 10 files changed, 185 insertions(+), 70 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl b/acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl new file mode 100644 index 00000000000..ef92f8119e7 --- /dev/null +++ b/acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# A job parameter references a volume's computed volume_path, and the volume's +# schema_name/name are references to another job's remote creator_user_name. So +# volume_path embeds a reference at initialize that only resolves at deploy, +# exercising the deferred-resolution chain (foo -> data.volume_path -> process) +# end to end with no drift afterwards. +resources: + jobs: + foo: + name: test-job-foo-$UNIQUE_NAME + + process: + name: test-job-process-$UNIQUE_NAME + parameters: + - name: data_path + default: ${resources.volumes.data.volume_path} + + volumes: + data: + catalog_name: main + schema_name: ${resources.jobs.foo.creator_user_name} + name: ${resources.jobs.foo.creator_user_name} diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index f68ec6b3111..72c978bd4aa 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -47,6 +47,7 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index e782f7ae269..824b2e859f6 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -65,6 +65,7 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl", ] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl index d9ac4031613..f716a20ded0 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl @@ -5,21 +5,24 @@ bundle: # This is the motivating use case from databricks/cli#4233: other resources can # refer to ${resources.volumes..volume_path} instead of hardcoding the UC # path. +# +# Here the volume's schema_name and name are themselves references to another +# job's remote creator_user_name, which is only known after deploy. So +# volume_path is computed at initialize with those references embedded and the +# whole chain (foo -> data.volume_path -> process) resolves at deploy. resources: - schemas: - my: - catalog_name: main - name: myschema-${UNIQUE_NAME} - - volumes: - data: - catalog_name: main - schema_name: myschema-${UNIQUE_NAME} - name: data-${UNIQUE_NAME} - jobs: + foo: + name: foo-${UNIQUE_NAME} + process: name: process-${UNIQUE_NAME} parameters: - name: data_path default: ${resources.volumes.data.volume_path} + + volumes: + data: + catalog_name: main + schema_name: ${resources.jobs.foo.creator_user_name} + name: ${resources.jobs.foo.creator_user_name} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json index 9599823840d..bf6b6289b68 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json @@ -1,3 +1,20 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + } +} { "method": "POST", "path": "/api/2.2/jobs/create", @@ -12,7 +29,7 @@ "name": "process-[UNIQUE_NAME]", "parameters": [ { - "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "default": "/Volumes/main/[USERNAME]/[USERNAME]", "name": "data_path" } ], diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json index 9599823840d..e69de29bb2d 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json @@ -1,23 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "process-[UNIQUE_NAME]", - "parameters": [ - { - "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", - "name": "data_path" - } - ], - "queue": { - "enabled": true - } - } -} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt index 5a4539339c5..2e94065e6b9 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt @@ -1,6 +1,40 @@ >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Error: exit status 1 + +Error: Unsupported attribute + + on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: + 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + +Error: Unsupported attribute + + on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: + 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + +Error: Unsupported attribute + + on bundle.tf.json line 51, in resource.databricks_volume.data: + 51: "name": "${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + +Error: Unsupported attribute + + on bundle.tf.json line 52, in resource.databricks_volume.data: + 52: "schema_name": "${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + + + +Exit code: 1 diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json index bc0c26ecb1c..a9bb0ed962c 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json @@ -4,7 +4,31 @@ "plan_version": 2, "cli_version": "[DEV_VERSION]", "plan": { + "resources.jobs.foo": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + } + } + }, "resources.jobs.process": { + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.creator_user_name}" + } + ], "action": "create", "new_state": { "value": { @@ -18,39 +42,37 @@ "name": "process-[UNIQUE_NAME]", "parameters": [ { - "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "default": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}", "name": "data_path" } ], "queue": { "enabled": true } - } - } - }, - "resources.schemas.my": { - "action": "create", - "new_state": { - "value": { - "catalog_name": "main", - "name": "myschema-[UNIQUE_NAME]" + }, + "vars": { + "parameters[0].default": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}" } } }, "resources.volumes.data": { "depends_on": [ { - "node": "resources.schemas.my", - "label": "${resources.schemas.my.name}" + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.creator_user_name}" } ], "action": "create", "new_state": { "value": { "catalog_name": "main", - "name": "data-[UNIQUE_NAME]", - "schema_name": "myschema-[UNIQUE_NAME]", + "name": "${resources.jobs.foo.creator_user_name}", + "schema_name": "${resources.jobs.foo.creator_user_name}", "volume_type": "MANAGED" + }, + "vars": { + "name": "${resources.jobs.foo.creator_user_name}", + "schema_name": "${resources.jobs.foo.creator_user_name}" } } } diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json index 90a3acec738..21582c82881 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json @@ -1,16 +1,39 @@ >>> errcode [CLI] bundle plan -o json -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.jobs.process": { - "action": "create" - }, - "resources.schemas.my": { - "action": "create" - }, - "resources.volumes.data": { - "action": "create" - } - } -} +Error: exit status 1 + +Error: Unsupported attribute + + on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: + 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + +Error: Unsupported attribute + + on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: + 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + +Error: Unsupported attribute + + on bundle.tf.json line 51, in resource.databricks_volume.data: + 51: "name": "${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + +Error: Unsupported attribute + + on bundle.tf.json line 52, in resource.databricks_volume.data: + 52: "schema_name": "${databricks_job.foo.creator_user_name}", + +This object has no argument, nested block, or exported attribute named +"creator_user_name". + + + +Exit code: 1 diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt index 55c41e798a4..63edb8decb4 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt @@ -1,6 +1,19 @@ >>> [CLI] bundle validate -o json { + "foo": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + }, "process": { "deployment": { "kind": "BUNDLE", @@ -12,7 +25,7 @@ "name": "process-[UNIQUE_NAME]", "parameters": [ { - "default": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "default": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}", "name": "data_path" } ], @@ -24,9 +37,9 @@ { "data": { "catalog_name": "main", - "name": "data-[UNIQUE_NAME]", - "schema_name": "${resources.schemas.my.name}", - "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/data-[UNIQUE_NAME]", + "name": "${resources.jobs.foo.creator_user_name}", + "schema_name": "${resources.jobs.foo.creator_user_name}", + "volume_path": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}", "volume_type": "MANAGED" } } From b3d678625b07737af5695c1e33d05443dfa0ca3b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 11:04:10 +0000 Subject: [PATCH 11/15] Fix lint and exclude volume_path_job_ref from terraform-based invariants Rewrite the state-type filter in extractReferences as a positive check to satisfy the nilerr linter. Exclude the volume_path_job_ref invariant config from the migrate and continue_293 suites: volume_path is unsupported by the v0.293.0 seed CLI, and its embedded creator_user_name reference is not a terraform-exported attribute. It remains covered by no_drift on direct. --- acceptance/bundle/invariant/continue_293/out.test.toml | 1 + acceptance/bundle/invariant/continue_293/test.toml | 4 ++++ acceptance/bundle/invariant/migrate/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 7 +++++++ bundle/direct/bundle_plan.go | 8 +++----- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 5c601542bce..ec5e8891bff 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -49,5 +49,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 6887ada9a71..686fe95257c 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -15,6 +15,10 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # Dotted pipeline configuration keys are not supported on v0.293.0 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"] +# Computed volume_path is not supported on v0.293.0, so the seed deploy cannot +# resolve ${resources.volumes.*.volume_path}. Covered by no_drift on direct. +EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"] + # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min # per variant (two full deploys at 1000 tasks) without incremental coverage. EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 5c601542bce..ec5e8891bff 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -49,5 +49,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index a121cbf6475..43da7a96ce8 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -30,3 +30,10 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # migrate deploys via Terraform first, and the TF provider rejects an uppercase # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] + +# volume_path embeds ${resources.jobs.foo.creator_user_name}; the terraform +# interpolator converts it to ${databricks_job.foo.creator_user_name}, but the +# terraform databricks_job resource does not export creator_user_name, so the +# terraform seed deploy fails. Same class as no_cross_resource_ref above. +# Covered by no_drift on direct. +EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"] diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5b8242ae274..009cead1414 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1077,12 +1077,10 @@ func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map // the state and must not be treated as a dependency here. Such references are // still made available to other resources that read the field (for example // ${resources.volumes.x.volume_path}) earlier during initialize. - if structaccess.ValidatePath(stateType, fieldPath) != nil { - return nil + if structaccess.ValidatePath(stateType, fieldPath) == nil { + // Store the original string that contains references, not individual references. + refs[fieldPath.String()] = ref.Str } - - // Store the original string that contains references, not individual references. - refs[fieldPath.String()] = ref.Str return nil }) if err != nil { From 9e1099879526956a034be8e317474a9f330c1320 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 12:08:55 +0000 Subject: [PATCH 12/15] Reject user-set volume_path, error on uncomputable references, document semantics InitializeVolumePaths now rejects a user-provided volume_path instead of silently overwriting the computed value, and ResolveVolumePathReferencesOnlyResources errors when a referenced volume_path could not be computed rather than injecting an empty string. Add a set-volume-path acceptance test for the rejection and expand the changelog to document the deploy-ordering and Terraform-engine caveats. --- NEXT_CHANGELOG.md | 2 ++ .../volumes/set-volume-path/databricks.yml | 12 +++++++ .../volumes/set-volume-path/out.requests.txt | 8 +++++ .../volumes/set-volume-path/out.test.toml | 3 ++ .../volumes/set-volume-path/output.txt | 13 +++++++ .../resources/volumes/set-volume-path/script | 1 + .../volumes/set-volume-path/test.toml | 2 ++ .../config/mutator/initialize_volume_paths.go | 9 ++++- .../mutator/initialize_volume_paths_test.go | 22 ++++++++++++ .../mutator/resolve_variable_references.go | 20 +++++++++-- .../resolve_variable_references_test.go | 11 +++--- bundle/direct/bundle_plan_test.go | 34 +++++++++++++++++++ 12 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/resources/volumes/set-volume-path/databricks.yml create mode 100644 acceptance/bundle/resources/volumes/set-volume-path/out.requests.txt create mode 100644 acceptance/bundle/resources/volumes/set-volume-path/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/set-volume-path/output.txt create mode 100644 acceptance/bundle/resources/volumes/set-volume-path/script create mode 100644 acceptance/bundle/resources/volumes/set-volume-path/test.toml diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 13977593d7f..2a8ca9693a3 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,6 +8,8 @@ ### Bundles * Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes..volume_path}` instead of hardcoding `/Volumes///` ([#5550](https://github.com/databricks/cli/pull/5550)). + * `volume_path` is derived purely from the volume's `catalog_name`, `schema_name`, and `name`, so the reference is resolved early (at initialize) and inlined into the referring field. Referencing `volume_path` therefore does not make the referring resource depend on the volume during deploy; if `catalog_name`/`schema_name`/`name` themselves reference other resources, the referrer depends on those resources instead. + * Supported on the direct deployment engine (`DATABRICKS_BUNDLE_ENGINE=direct`). On the Terraform engine `volume_path` is dropped before apply, and a `volume_path` whose components embed a value only known after deploy (for example `${resources.jobs..creator_user_name}`) is not supported. ### Dependency updates diff --git a/acceptance/bundle/resources/volumes/set-volume-path/databricks.yml b/acceptance/bundle/resources/volumes/set-volume-path/databricks.yml new file mode 100644 index 00000000000..19b19377fd8 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +# volume_path is computed and read-only. Setting it explicitly must be rejected +# rather than silently overwritten with the computed path. +resources: + volumes: + volume1: + catalog_name: main + schema_name: myschema + name: myvolume + volume_path: /Volumes/bogus/path/set-by-user diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.requests.txt b/acceptance/bundle/resources/volumes/set-volume-path/out.requests.txt new file mode 100644 index 00000000000..12576f503d5 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.requests.txt @@ -0,0 +1,8 @@ +{ + "method": "GET", + "path": "/.well-known/databricks-config" +} +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-volume-path/output.txt b/acceptance/bundle/resources/volumes/set-volume-path/output.txt new file mode 100644 index 00000000000..a312ea1b1de --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/output.txt @@ -0,0 +1,13 @@ + +>>> errcode [CLI] bundle validate +Error: resources.volumes.volume1.volume_path is computed and read-only; remove it from the configuration + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/resources/volumes/set-volume-path/script b/acceptance/bundle/resources/volumes/set-volume-path/script new file mode 100644 index 00000000000..9ecda517f9b --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/script @@ -0,0 +1 @@ +trace errcode $CLI bundle validate diff --git a/acceptance/bundle/resources/volumes/set-volume-path/test.toml b/acceptance/bundle/resources/volumes/set-volume-path/test.toml new file mode 100644 index 00000000000..7d36fb9dc18 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/test.toml @@ -0,0 +1,2 @@ +Local = true +Cloud = false diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go index b58c04fcf89..29939f9ef96 100644 --- a/bundle/config/mutator/initialize_volume_paths.go +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -2,6 +2,7 @@ package mutator import ( "context" + "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" @@ -36,7 +37,13 @@ func (m *initializeVolumePaths) Name() string { func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("volumes"), dyn.AnyKey()) - return dyn.MapByPattern(root, pattern, func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // volume_path is computed and read-only. Reject a user-provided value + // instead of silently overwriting it with the computed path. + if existing, ok := v.Get("volume_path").AsString(); ok && existing != "" { + return dyn.InvalidValue, fmt.Errorf("%s.volume_path is computed and read-only; remove it from the configuration", p.String()) + } + var vol resources.Volume if err := convert.ToTyped(&vol, v); err != nil { return dyn.InvalidValue, err diff --git a/bundle/config/mutator/initialize_volume_paths_test.go b/bundle/config/mutator/initialize_volume_paths_test.go index e680441a692..c2816d0642e 100644 --- a/bundle/config/mutator/initialize_volume_paths_test.go +++ b/bundle/config/mutator/initialize_volume_paths_test.go @@ -81,6 +81,28 @@ func TestInitializeVolumePaths_UnresolvedReference(t *testing.T) { require.Equal(t, "${resources.schemas.missing.name}", b.Config.Resources.Volumes["foo"].SchemaName) } +func TestInitializeVolumePaths_RejectsUserProvidedPath(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volfoo", + }, + VolumePath: "/Volumes/bogus/path/set-by-user", + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.ErrorContains(t, diags.Error(), "volume_path is computed and read-only") +} + func TestInitializeVolumePaths_MalformedReference(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index b43777f6e34..9f3a45c01ee 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -99,13 +99,30 @@ func ResolveVariableReferencesInLookup() bundle.Mutator { func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { return &resolveVariableReferences{ prefixes: []string{"resources"}, - lookupFn: lookup, + lookupFn: lookupVolumePath, allowPathFn: isVolumePathReferencePath, extraRounds: maxResolutionRounds - 1, includeResources: true, } } +// lookupVolumePath resolves a reference to resources.volumes..volume_path. +// +// A volume's volume_path is empty only when it could not be computed, for example because +// catalog_name, schema_name, or name is unset or contains a malformed reference (see +// Volume.ComputeVolumePath). Resolving such a reference to "" would silently inject an empty +// string into the referrer, so we return an actionable error instead. +func lookupVolumePath(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { + result, err := lookup(v, path, b) + if err != nil { + return dyn.InvalidValue, err + } + if s, ok := result.AsString(); ok && s == "" { + return dyn.InvalidValue, fmt.Errorf("cannot resolve ${%s}: volume_path could not be computed; set catalog_name, schema_name, and name and ensure they contain no malformed references", path.String()) + } + return result, nil +} + func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { if path.String() == "workspace.file_path" { @@ -330,6 +347,5 @@ func isVolumePathReferencePath(path dyn.Path) bool { } return path[0].Key() == "resources" && path[1].Key() == "volumes" && - path[2].Key() != "" && path[3].Key() == "volume_path" } diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 5d9646b7f22..d8e7ca931db 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -105,10 +105,10 @@ func TestResolveVolumePathReferencesOnlyResources_MissingTarget(t *testing.T) { require.ErrorContains(t, diags.Error(), "reference does not exist: ${resources.volumes.missing.volume_path}") } -func TestResolveVolumePathReferencesOnlyResources_UnsetTargetResolvesToEmpty(t *testing.T) { - // When the target volume exists but its volume_path was never computed (for - // example because its name is only known at deploy), the field is normalized - // to an empty string, so the reference resolves to "" rather than erroring. +func TestResolveVolumePathReferencesOnlyResources_UncomputedTargetErrors(t *testing.T) { + // When the target volume exists but its volume_path could not be computed (for + // example because catalog_name/schema_name/name is unset), the reference must + // error rather than silently resolving to an empty string. b := &bundle.Bundle{ Config: config.Root{ Resources: config.Resources{ @@ -122,8 +122,7 @@ func TestResolveVolumePathReferencesOnlyResources_UnsetTargetResolvesToEmpty(t * b.Config.Resources.Volumes["ref"].Comment = "${resources.volumes.foo.volume_path}" diags := bundle.Apply(t.Context(), b, ResolveVolumePathReferencesOnlyResources()) - require.NoError(t, diags.Error()) - require.Empty(t, b.Config.Resources.Volumes["ref"].Comment) + require.ErrorContains(t, diags.Error(), "volume_path could not be computed") } func TestIsVolumePathReferencePath(t *testing.T) { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index c7f6e8dbf36..a098da7561c 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -1,11 +1,13 @@ package direct import ( + "bytes" "testing" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,6 +42,38 @@ func TestDynPathToStructPath(t *testing.T) { } } +// extractReferences gates references on the resource's state type, so a reference +// stored in a field that exists in the input config but not in state (most notably a +// bundle:"readonly" field such as volumes' computed volume_path) must not become a +// dependency. References in state fields (e.g. comment) are still extracted. +func TestExtractReferences_ExcludesReadonlyFields(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + // The volume state type is catalog.CreateVolumeRequestContent, which has + // comment but not volume_path. + stateType := adapters["volumes"].StateType() + + const yml = ` +resources: + volumes: + v: + catalog_name: main + schema_name: myschema + name: myvol + comment: "${resources.schemas.kept.name}" + volume_path: "/Volumes/main/${resources.schemas.dropped.name}/myvol" +` + root, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yml)) + require.NoError(t, err) + + refs, err := extractReferences(root, "resources.volumes.v", stateType) + require.NoError(t, err) + + assert.Equal(t, map[string]string{ + "comment": "${resources.schemas.kept.name}", + }, refs) +} + func TestShouldSkipBackendDefault_ManagedPropertiesOnly(t *testing.T) { // Rules mirror the schemas backend_defaults in resources.yml, but the test is // deliberately self-contained so that edits to resources.yml don't break it. From b7639296982ec0c5e9bb60e0a956e0367272b404 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 12:42:14 +0000 Subject: [PATCH 13/15] Clarify volume_path doc comments Update ExtractReferences doc to reflect state-type filtering of references, correct the InitializeVolumePaths interpolation comment to point at the resolver mutator, and note the run-exactly-once requirement. --- bundle/config/mutator/initialize_volume_paths.go | 9 ++++++--- bundle/direct/bundle_plan.go | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go index 29939f9ef96..0dff805f37b 100644 --- a/bundle/config/mutator/initialize_volume_paths.go +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -23,9 +23,12 @@ type initializeVolumePaths struct{} // // A component that cannot be resolved locally (for example a remote field only known at plan or // deploy time) is left as a ${...} reference and embedded into volume_path. The embedded -// reference is then carried through ${resources.volumes..volume_path} interpolation and -// resolved later by the engine during plan or deploy, the same way any other resource reference -// is. This enables ${resources.volumes..volume_path} interpolation during initialize. +// reference is then carried into any ${resources.volumes..volume_path} referrer (resolved +// by ResolveVolumePathReferencesOnlyResources) and ultimately resolved by the engine during plan +// or deploy, the same way any other resource reference is. +// +// This mutator must run exactly once: volume_path is computed here and never persisted, so a +// second run would see the value it set and trip the "computed and read-only" rejection below. func InitializeVolumePaths() bundle.Mutator { return &initializeVolumePaths{} } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 009cead1414..f5a118ebffd 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1030,7 +1030,9 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } -// ExtractReferences extracts all variable references from the config subtree rooted at node. +// ExtractReferences extracts variable references from the config subtree rooted at node, +// keeping only those whose field path exists in stateType (references in input-only or +// bundle:"readonly" fields, such as volumes' computed volume_path, are skipped). // Returns a map from structpath string (field path within the resource) to template string. func ExtractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { return extractReferences(root, node, stateType) From af3128950dd7963446d425ad531989ffb6e775c6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 17:16:47 +0000 Subject: [PATCH 14/15] Condense volume_path changelog entry into a single bullet --- NEXT_CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 2a8ca9693a3..6c182bb5df9 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,9 +7,7 @@ ### CLI ### Bundles -* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes..volume_path}` instead of hardcoding `/Volumes///` ([#5550](https://github.com/databricks/cli/pull/5550)). - * `volume_path` is derived purely from the volume's `catalog_name`, `schema_name`, and `name`, so the reference is resolved early (at initialize) and inlined into the referring field. Referencing `volume_path` therefore does not make the referring resource depend on the volume during deploy; if `catalog_name`/`schema_name`/`name` themselves reference other resources, the referrer depends on those resources instead. - * Supported on the direct deployment engine (`DATABRICKS_BUNDLE_ENGINE=direct`). On the Terraform engine `volume_path` is dropped before apply, and a `volume_path` whose components embed a value only known after deploy (for example `${resources.jobs..creator_user_name}`) is not supported. +* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes..volume_path}` instead of hardcoding `/Volumes///` ([#5550](https://github.com/databricks/cli/pull/5550)). Derived from `catalog_name`/`schema_name`/`name` and resolved at initialize, so the reference depends on those underlying resources rather than the volume itself. Direct engine only; on Terraform it is dropped before apply and components only known after deploy are unsupported. ### Dependency updates From 36ac48e75dcbb9a0e0d522af55acb0ae23afe17f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 25 Jun 2026 17:44:11 +0000 Subject: [PATCH 15/15] Restrict volume_path_job_ref to direct engine; clarify changelog The remote-field embedding scenario (volume_path carrying an after-deploy ${...creator_user_name}) is only supported on the direct engine, which is the default. Recording Terraform's provider-level failure as expected output was brittle, so run this case on direct only and drop the stale Terraform artifacts. Correct the changelog: ${...volume_path} references resolve on both engines when components are known at initialize; only after-deploy components are direct-only. --- NEXT_CHANGELOG.md | 2 +- .../volume_path_job_ref/databricks.yml.tmpl | 4 ++ .../out.deploy.requests.terraform.json | 0 .../out.deploy.terraform.txt | 40 ------------------- .../out.plan.terraform.json | 39 ------------------ .../volume_path_job_ref/out.test.toml | 2 +- .../volume_path_job_ref/test.toml | 7 ++++ 7 files changed, 13 insertions(+), 81 deletions(-) delete mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json delete mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt delete mode 100644 acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 6c182bb5df9..2e9a9555d85 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,7 +7,7 @@ ### CLI ### Bundles -* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes..volume_path}` instead of hardcoding `/Volumes///` ([#5550](https://github.com/databricks/cli/pull/5550)). Derived from `catalog_name`/`schema_name`/`name` and resolved at initialize, so the reference depends on those underlying resources rather than the volume itself. Direct engine only; on Terraform it is dropped before apply and components only known after deploy are unsupported. +* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes..volume_path}` instead of hardcoding `/Volumes///` ([#5550](https://github.com/databricks/cli/pull/5550)). Derived from `catalog_name`/`schema_name`/`name` and resolved at initialize, so the reference depends on those underlying resources rather than the volume itself. The field is computed by the CLI and never sent to the API (dropped before Terraform apply). References resolve on both engines when the path components are known at initialize; components that are only known after deploy (for example a remote `creator_user_name`) are supported on the direct engine (the default) but not on Terraform. ### Dependency updates diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl index f716a20ded0..e4bc6a10eef 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl @@ -10,6 +10,10 @@ bundle: # job's remote creator_user_name, which is only known after deploy. So # volume_path is computed at initialize with those references embedded and the # whole chain (foo -> data.volume_path -> process) resolves at deploy. +# +# This is direct-engine only (the default): Terraform cannot export +# creator_user_name as an attribute, so the embedded reference is unsupported +# there. See test.toml for the engine restriction. resources: jobs: foo: diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.terraform.json deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt deleted file mode 100644 index 2e94065e6b9..00000000000 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.terraform.txt +++ /dev/null @@ -1,40 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... -Error: exit status 1 - -Error: Unsupported attribute - - on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: - 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - -Error: Unsupported attribute - - on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: - 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - -Error: Unsupported attribute - - on bundle.tf.json line 51, in resource.databricks_volume.data: - 51: "name": "${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - -Error: Unsupported attribute - - on bundle.tf.json line 52, in resource.databricks_volume.data: - 52: "schema_name": "${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - - - -Exit code: 1 diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json deleted file mode 100644 index 21582c82881..00000000000 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.terraform.json +++ /dev/null @@ -1,39 +0,0 @@ - ->>> errcode [CLI] bundle plan -o json -Error: exit status 1 - -Error: Unsupported attribute - - on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: - 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - -Error: Unsupported attribute - - on bundle.tf.json line 39, in resource.databricks_job.process.parameter[0]: - 39: "default": "/Volumes/main/${databricks_job.foo.creator_user_name}/${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - -Error: Unsupported attribute - - on bundle.tf.json line 51, in resource.databricks_volume.data: - 51: "name": "${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - -Error: Unsupported attribute - - on bundle.tf.json line 52, in resource.databricks_volume.data: - 52: "schema_name": "${databricks_job.foo.creator_user_name}", - -This object has no argument, nested block, or exported attribute named -"creator_user_name". - - - -Exit code: 1 diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index f784a183258..e90b6d5d1ba 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml index 159efe02696..77681b7380c 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -1 +1,8 @@ RecordRequests = true + +# Direct-engine only: this scenario embeds a reference to a remote field +# (jobs.foo.creator_user_name, known only after deploy) into volume_path. The +# direct engine resolves it at deploy; Terraform cannot export that attribute, +# so the case is unsupported there (see NEXT_CHANGELOG.md). Direct is the +# default engine, so the motivating use case works out of the box. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]