diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 13efb33eb74..151acf4932d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI ### Bundles +* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes..volume_path}` instead of hardcoding `/Volumes///` ([#5550](https://github.com/databricks/cli/pull/5550)). Derived from `catalog_name`/`schema_name`/`name` and resolved at initialize, so the reference depends on those underlying resources rather than the volume itself. The field is computed by the CLI and never sent to the API (dropped before Terraform apply). References resolve on both engines when the path components are known at initialize; components that are only known after deploy (for example a remote `creator_user_name`) are supported on the direct engine (the default) but not on Terraform. * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)). diff --git a/acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl b/acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl new file mode 100644 index 00000000000..ef92f8119e7 --- /dev/null +++ b/acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# A job parameter references a volume's computed volume_path, and the volume's +# schema_name/name are references to another job's remote creator_user_name. So +# volume_path embeds a reference at initialize that only resolves at deploy, +# exercising the deferred-resolution chain (foo -> data.volume_path -> process) +# end to end with no drift afterwards. +resources: + jobs: + foo: + name: test-job-foo-$UNIQUE_NAME + + process: + name: test-job-process-$UNIQUE_NAME + parameters: + - name: data_path + default: ${resources.volumes.data.volume_path} + + volumes: + data: + catalog_name: main + schema_name: ${resources.jobs.foo.creator_user_name} + name: ${resources.jobs.foo.creator_user_name} diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 54b97c6301f..af5248dc9a3 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -16,6 +16,10 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # Dotted pipeline configuration keys are not supported on v0.293.0 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"] +# Computed volume_path is not supported on v0.293.0, so the seed deploy cannot +# resolve ${resources.volumes.*.volume_path}. Covered by no_drift on direct. +EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"] + # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min # per variant (two full deploys at 1000 tasks) without incremental coverage. EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index a121cbf6475..43da7a96ce8 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -30,3 +30,10 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # migrate deploys via Terraform first, and the TF provider rejects an uppercase # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] + +# volume_path embeds ${resources.jobs.foo.creator_user_name}; the terraform +# interpolator converts it to ${databricks_job.foo.creator_user_name}, but the +# terraform databricks_job resource does not export creator_user_name, so the +# terraform seed deploy fails. Same class as no_cross_resource_ref above. +# Covered by no_drift on direct. +EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index e39444592d5..af8014ad7a2 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -49,6 +49,7 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl" ] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index c0ffd515828..890d5c70a2c 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -67,6 +67,7 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", "volume_uppercase_name.yml.tmpl", ] diff --git a/acceptance/bundle/python/volumes-support/output.txt b/acceptance/bundle/python/volumes-support/output.txt index e2b6d8771e6..eafa1e8685d 100644 --- a/acceptance/bundle/python/volumes-support/output.txt +++ b/acceptance/bundle/python/volumes-support/output.txt @@ -17,12 +17,14 @@ "catalog_name": "my_catalog", "name": "My Volume (updated)", "schema_name": "my_schema", + "volume_path": "/Volumes/my_catalog/my_schema/My Volume (updated)", "volume_type": "MANAGED" }, "my_volume_2": { "catalog_name": "my_catalog_2", "name": "My Volume (2) (updated)", "schema_name": "my_schema_2", + "volume_path": "/Volumes/my_catalog_2/my_schema_2/My Volume (2) (updated)", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 7d7b4913c9d..487ab6a1de9 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3476,6 +3476,7 @@ resources.volumes.*.updated_at int64 REMOTE resources.volumes.*.updated_by string REMOTE resources.volumes.*.url string INPUT resources.volumes.*.volume_id string REMOTE +resources.volumes.*.volume_path string INPUT resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL resources.volumes.*.grants.securable_type string ALL diff --git a/acceptance/bundle/resource_deps/bad_syntax/output.txt b/acceptance/bundle/resource_deps/bad_syntax/output.txt index 0e9d83b6431..d5bde99d707 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/output.txt +++ b/acceptance/bundle/resource_deps/bad_syntax/output.txt @@ -6,6 +6,7 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { diff --git a/acceptance/bundle/resource_deps/computed_volume_path/databricks.yml.tmpl b/acceptance/bundle/resource_deps/computed_volume_path/databricks.yml.tmpl new file mode 100644 index 00000000000..ec1e2619848 --- /dev/null +++ b/acceptance/bundle/resource_deps/computed_volume_path/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: testbundle-${UNIQUE_NAME} + +resources: + schemas: + my: + catalog_name: main + name: myschema-${UNIQUE_NAME} + + volumes: + bar: + catalog_name: main + schema_name: myschema-${UNIQUE_NAME} + name: volumebar-${UNIQUE_NAME} + + foo: + catalog_name: main + schema_name: myschema-${UNIQUE_NAME} + name: volumefoo-${UNIQUE_NAME} + comment: ${resources.volumes.bar.volume_path} diff --git a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/computed_volume_path/output.txt b/acceptance/bundle/resource_deps/computed_volume_path/output.txt new file mode 100644 index 00000000000..2db063aca34 --- /dev/null +++ b/acceptance/bundle/resource_deps/computed_volume_path/output.txt @@ -0,0 +1,36 @@ + +>>> [CLI] bundle validate -o json +{ + "schemas": { + "my": { + "catalog_name": "main", + "name": "myschema-[UNIQUE_NAME]" + } + }, + "volumes": { + "bar": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", + "volume_type": "MANAGED" + }, + "foo": { + "catalog_name": "main", + "comment": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + } +} + +>>> [CLI] bundle plan +create schemas.my +create volumes.bar +create volumes.foo + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged + +>>> print_requests.py --get //unity diff --git a/acceptance/bundle/resource_deps/computed_volume_path/script b/acceptance/bundle/resource_deps/computed_volume_path/script new file mode 100644 index 00000000000..c6c051b389b --- /dev/null +++ b/acceptance/bundle/resource_deps/computed_volume_path/script @@ -0,0 +1,4 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle plan +trace print_requests.py --get //unity diff --git a/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt b/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt index 2e454e81522..4258182b523 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt +++ b/acceptance/bundle/resource_deps/implicit_deps_volume/output.txt @@ -17,6 +17,7 @@ "catalog_name": "${resources.catalogs.my_catalog.name}", "name": "myvolume", "schema_name": "${resources.schemas.my_schema.name}", + "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/non_existent_field/output.txt b/acceptance/bundle/resource_deps/non_existent_field/output.txt index ec7de848680..09d3f03d6d7 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/output.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/output.txt @@ -6,12 +6,14 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { "catalog_name": "${resources.volumes.bar.non_existent}", "name": "myname", "schema_name": "${resources.volumes.bar.schema_name}", + "volume_path": "/Volumes/${resources.volumes.bar.non_existent}/myschema/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt b/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt index ce5fcfb66ba..51bd4b30bc5 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt @@ -12,6 +12,7 @@ "catalog_name": "main", "name": "volumebar-[UNIQUE_NAME]", "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", "volume_type": "MANAGED" }, "foo": { @@ -19,6 +20,7 @@ "comment": "${resources.volumes.bar.storage_location}", "name": "volumefoo-[UNIQUE_NAME]", "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index 34e6a69b57a..3edca4ad787 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -21,12 +21,14 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { "catalog_name": "${resources.volumes.bar.catalog_name}", "name": "myname", "schema_name": "${resources.volumes.bar.schema_name}", + "volume_path": "/Volumes/mycatalog/myschema/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt index 87533e080e7..08dbf899f70 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt @@ -27,6 +27,7 @@ "catalog_name": "${resources.schemas.bar.catalog_name}", "name": "myname", "schema_name": "${resources.schemas.bar.name}", + "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/databricks.yml b/acceptance/bundle/resource_deps/volume_path_contains_id/databricks.yml new file mode 100644 index 00000000000..190e9411d01 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/databricks.yml @@ -0,0 +1,29 @@ +bundle: + name: test-bundle + +# volume_path is a CLI-only computed field (not API state). When a component +# (catalog_name, schema_name, name) is an unresolved ${...} reference, it stays +# embedded in volume_path so it resolves later at plan/deploy. +# +# Chain under test: bar.name is a remote value (baz.id), so bar.volume_path is +# computed at initialize with that reference intact; foo.comment picks up that +# string and resolves at deploy to the fully-resolved path. +# +# baz.id is used instead of baz.storage_location because the resolved value +# becomes bar's name, and the testserver can't route a name containing the +# slashes of a storage_location URL. +resources: + volumes: + baz: + catalog_name: mycatalog + schema_name: myschema + name: bazname + bar: + catalog_name: mycatalog + schema_name: myschema + name: ${resources.volumes.baz.id} + foo: + catalog_name: mycatalog + schema_name: myschema + name: fooname + comment: ${resources.volumes.bar.volume_path} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.direct.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.direct.txt new file mode 100644 index 00000000000..0fd05668156 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.direct.json new file mode 100644 index 00000000000..df6294fddfc --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.direct.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname", + "name": "fooname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "mycatalog.myschema.bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.terraform.json new file mode 100644 index 00000000000..a8cb2d0afcd --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.terraform.json @@ -0,0 +1,43 @@ +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname", + "name": "fooname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "mycatalog", + "name": "mycatalog.myschema.bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.terraform.txt new file mode 100644 index 00000000000..0fd05668156 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.terraform.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.direct.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.direct.txt new file mode 100644 index 00000000000..a60dfafafad --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.direct.txt @@ -0,0 +1,17 @@ +The following resources will be deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.terraform.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.terraform.txt new file mode 100644 index 00000000000..a60dfafafad --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.terraform.txt @@ -0,0 +1,17 @@ +The following resources will be deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.bar + delete resources.volumes.baz + delete resources.volumes.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.direct.json new file mode 100644 index 00000000000..b7a38b0bbb4 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.direct.json @@ -0,0 +1,24 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.terraform.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.terraform.json new file mode 100644 index 00000000000..b7a38b0bbb4 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy_requests.terraform.json @@ -0,0 +1,24 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname" +} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json new file mode 100644 index 00000000000..fd53a596773 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json @@ -0,0 +1,58 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.volumes.bar": { + "depends_on": [ + { + "node": "resources.volumes.baz", + "label": "${resources.volumes.baz.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "mycatalog", + "name": "${resources.volumes.baz.id}", + "schema_name": "myschema", + "volume_type": "MANAGED" + }, + "vars": { + "name": "${resources.volumes.baz.id}" + } + } + }, + "resources.volumes.baz": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + } + }, + "resources.volumes.foo": { + "depends_on": [ + { + "node": "resources.volumes.baz", + "label": "${resources.volumes.baz.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "mycatalog", + "comment": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}", + "name": "fooname", + "schema_name": "myschema", + "volume_type": "MANAGED" + }, + "vars": { + "comment": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.terraform.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.terraform.json new file mode 100644 index 00000000000..44882021fba --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.terraform.json @@ -0,0 +1,14 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.volumes.bar": { + "action": "create" + }, + "resources.volumes.baz": { + "action": "create" + }, + "resources.volumes.foo": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/output.txt b/acceptance/bundle/resource_deps/volume_path_contains_id/output.txt new file mode 100644 index 00000000000..8cbd00ede13 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle validate -o json +{ + "volumes": { + "bar": { + "catalog_name": "mycatalog", + "name": "${resources.volumes.baz.id}", + "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}", + "volume_type": "MANAGED" + }, + "baz": { + "catalog_name": "mycatalog", + "name": "bazname", + "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/bazname", + "volume_type": "MANAGED" + }, + "foo": { + "catalog_name": "mycatalog", + "comment": "/Volumes/mycatalog/myschema/${resources.volumes.baz.id}", + "name": "fooname", + "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/fooname", + "volume_type": "MANAGED" + } + } +} + +>>> [CLI] bundle plan +create volumes.bar +create volumes.baz +create volumes.foo + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/script b/acceptance/bundle/resource_deps/volume_path_contains_id/script new file mode 100644 index 00000000000..97b2c4d693e --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/script @@ -0,0 +1,16 @@ +cleanup() { + $CLI bundle destroy --auto-approve &> out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt + print_requests.py --sort --get //unity &> out.destroy_requests.$DATABRICKS_BUNDLE_ENGINE.json +} + +trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle plan +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +trap cleanup EXIT +trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + +# bar.name is created from baz.id (resolved during deploy), and foo.comment is +# created with bar.volume_path, whose embedded baz.id reference also resolves +# during deploy: +print_requests.py --sort --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl new file mode 100644 index 00000000000..e4bc6a10eef --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/databricks.yml.tmpl @@ -0,0 +1,32 @@ +bundle: + name: testbundle-${UNIQUE_NAME} + +# A job parameter references a bundle-managed volume's computed volume_path. +# This is the motivating use case from databricks/cli#4233: other resources can +# refer to ${resources.volumes..volume_path} instead of hardcoding the UC +# path. +# +# Here the volume's schema_name and name are themselves references to another +# job's remote creator_user_name, which is only known after deploy. So +# volume_path is computed at initialize with those references embedded and the +# whole chain (foo -> data.volume_path -> process) resolves at deploy. +# +# This is direct-engine only (the default): Terraform cannot export +# creator_user_name as an attribute, so the embedded reference is unsupported +# there. See test.toml for the engine restriction. +resources: + jobs: + foo: + name: foo-${UNIQUE_NAME} + + process: + name: process-${UNIQUE_NAME} + parameters: + - name: data_path + default: ${resources.volumes.data.volume_path} + + volumes: + data: + catalog_name: main + schema_name: ${resources.jobs.foo.creator_user_name} + name: ${resources.jobs.foo.creator_user_name} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.direct.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.direct.txt new file mode 100644 index 00000000000..5a4539339c5 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json new file mode 100644 index 00000000000..bf6b6289b68 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.deploy.requests.direct.json @@ -0,0 +1,40 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "process-[UNIQUE_NAME]", + "parameters": [ + { + "default": "/Volumes/main/[USERNAME]/[USERNAME]", + "name": "data_path" + } + ], + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json new file mode 100644 index 00000000000..a9bb0ed962c --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json @@ -0,0 +1,80 @@ + +>>> errcode [CLI] bundle plan -o json +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.foo": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + } + } + }, + "resources.jobs.process": { + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.creator_user_name}" + } + ], + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "process-[UNIQUE_NAME]", + "parameters": [ + { + "default": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}", + "name": "data_path" + } + ], + "queue": { + "enabled": true + } + }, + "vars": { + "parameters[0].default": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}" + } + } + }, + "resources.volumes.data": { + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.creator_user_name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "${resources.jobs.foo.creator_user_name}", + "schema_name": "${resources.jobs.foo.creator_user_name}", + "volume_type": "MANAGED" + }, + "vars": { + "name": "${resources.jobs.foo.creator_user_name}", + "schema_name": "${resources.jobs.foo.creator_user_name}" + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt new file mode 100644 index 00000000000..63edb8decb4 --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt @@ -0,0 +1,47 @@ + +>>> [CLI] bundle validate -o json +{ + "foo": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + }, + "process": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "process-[UNIQUE_NAME]", + "parameters": [ + { + "default": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}", + "name": "data_path" + } + ], + "queue": { + "enabled": true + } + } +} +{ + "data": { + "catalog_name": "main", + "name": "${resources.jobs.foo.creator_user_name}", + "schema_name": "${resources.jobs.foo.creator_user_name}", + "volume_path": "/Volumes/main/${resources.jobs.foo.creator_user_name}/${resources.jobs.foo.creator_user_name}", + "volume_type": "MANAGED" + } +} + +>>> print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script new file mode 100644 index 00000000000..bcc1cf937cb --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -0,0 +1,12 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle validate -o json | jq '.resources.jobs, .resources.volumes' + +# The job's data_path default references the volume's computed volume_path. +# Record the JSON plan per engine to confirm the reference is resolved into the +# planned new_state rather than left as ${resources...}. +trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +# Deploy and inspect the create-job request: the parameter default sent to +# the Jobs API must be the interpolated volume_path. +trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py //jobs > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml new file mode 100644 index 00000000000..77681b7380c --- /dev/null +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -0,0 +1,8 @@ +RecordRequests = true + +# Direct-engine only: this scenario embeds a reference to a remote field +# (jobs.foo.creator_user_name, known only after deploy) into volume_path. The +# direct engine resolves it at deploy; Terraform cannot export that attribute, +# so the case is unsupported there (see NEXT_CHANGELOG.md). Direct is the +# default engine, so the motivating use case works out of the box. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/change-comment/output.txt b/acceptance/bundle/resources/volumes/change-comment/output.txt index aeba16f2f30..4b0aed08f37 100644 --- a/acceptance/bundle/resources/volumes/change-comment/output.txt +++ b/acceptance/bundle/resources/volumes/change-comment/output.txt @@ -7,6 +7,7 @@ "modified_status": "created", "name": "myvolume", "schema_name": "myschema", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } @@ -65,6 +66,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } @@ -116,5 +118,6 @@ Error: Resource catalog.VolumeInfo not found: main.myschema.myvolume "modified_status": "created", "name": "myvolume", "schema_name": "myschema", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/resources/volumes/change-name/output.txt b/acceptance/bundle/resources/volumes/change-name/output.txt index 3a920af70a3..8c37d76b92b 100644 --- a/acceptance/bundle/resources/volumes/change-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-name/output.txt @@ -28,6 +28,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resources/volumes/change-schema-name/output.txt b/acceptance/bundle/resources/volumes/change-schema-name/output.txt index bd196b0fb7e..9e0983508e9 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-schema-name/output.txt @@ -28,6 +28,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resources/volumes/set-volume-path/databricks.yml b/acceptance/bundle/resources/volumes/set-volume-path/databricks.yml new file mode 100644 index 00000000000..19b19377fd8 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +# volume_path is computed and read-only. Setting it explicitly must be rejected +# rather than silently overwritten with the computed path. +resources: + volumes: + volume1: + catalog_name: main + schema_name: myschema + name: myvolume + volume_path: /Volumes/bogus/path/set-by-user diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.requests.txt b/acceptance/bundle/resources/volumes/set-volume-path/out.requests.txt new file mode 100644 index 00000000000..12576f503d5 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.requests.txt @@ -0,0 +1,8 @@ +{ + "method": "GET", + "path": "/.well-known/databricks-config" +} +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-volume-path/output.txt b/acceptance/bundle/resources/volumes/set-volume-path/output.txt new file mode 100644 index 00000000000..a312ea1b1de --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/output.txt @@ -0,0 +1,13 @@ + +>>> errcode [CLI] bundle validate +Error: resources.volumes.volume1.volume_path is computed and read-only; remove it from the configuration + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/resources/volumes/set-volume-path/script b/acceptance/bundle/resources/volumes/set-volume-path/script new file mode 100644 index 00000000000..9ecda517f9b --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/script @@ -0,0 +1 @@ +trace errcode $CLI bundle validate diff --git a/acceptance/bundle/resources/volumes/set-volume-path/test.toml b/acceptance/bundle/resources/volumes/set-volume-path/test.toml new file mode 100644 index 00000000000..7d36fb9dc18 --- /dev/null +++ b/acceptance/bundle/resources/volumes/set-volume-path/test.toml @@ -0,0 +1,2 @@ +Local = true +Cloud = false diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index 08579b5dd2b..f7d3e8d2053 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -49,6 +49,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } @@ -104,6 +105,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } @@ -159,6 +161,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json index cf1745cc2f0..258b6b938ab 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json index cf1745cc2f0..258b6b938ab 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json index eff500341a0..3fe9aeea34a 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json @@ -57,6 +57,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json index cf1745cc2f0..258b6b938ab 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go new file mode 100644 index 00000000000..0dff805f37b --- /dev/null +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -0,0 +1,92 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +type initializeVolumePaths struct{} + +// InitializeVolumePaths sets resources.volumes.*.volume_path from catalog, schema, and name. +// +// catalog_name and schema_name may be ${resources.catalogs/schemas..name} references +// (for example, after CaptureUCDependencies rewrites implicit dependencies). We resolve those +// references locally to compute the path, but we do not write the resolved values back: the +// original references are preserved so validate and plan keep showing them. +// +// A component that cannot be resolved locally (for example a remote field only known at plan or +// deploy time) is left as a ${...} reference and embedded into volume_path. The embedded +// reference is then carried into any ${resources.volumes..volume_path} referrer (resolved +// by ResolveVolumePathReferencesOnlyResources) and ultimately resolved by the engine during plan +// or deploy, the same way any other resource reference is. +// +// This mutator must run exactly once: volume_path is computed here and never persisted, so a +// second run would see the value it set and trip the "computed and read-only" rejection below. +func InitializeVolumePaths() bundle.Mutator { + return &initializeVolumePaths{} +} + +func (m *initializeVolumePaths) Name() string { + return "InitializeVolumePaths" +} + +func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("volumes"), dyn.AnyKey()) + return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // volume_path is computed and read-only. Reject a user-provided value + // instead of silently overwriting it with the computed path. + if existing, ok := v.Get("volume_path").AsString(); ok && existing != "" { + return dyn.InvalidValue, fmt.Errorf("%s.volume_path is computed and read-only; remove it from the configuration", p.String()) + } + + var vol resources.Volume + if err := convert.ToTyped(&vol, v); err != nil { + return dyn.InvalidValue, err + } + + // Resolve references for the purpose of computing the path only; the + // original field values in v are left untouched. + vol.CatalogName = resolveResourceReference(root, vol.CatalogName) + vol.SchemaName = resolveResourceReference(root, vol.SchemaName) + vol.Name = resolveResourceReference(root, vol.Name) + + path := vol.ComputeVolumePath() + if path == "" { + return v, nil + } + return dyn.Set(v, "volume_path", dyn.V(path)) + }) + }) + if err != nil { + return diag.FromErr(err) + } + return nil +} + +// resolveResourceReference returns the concrete value of a pure ${resources....} reference +// by looking it up in root. Non-reference values and references that cannot be resolved (or +// resolve to another reference) are returned unchanged, so they still contain "${" and the +// caller will not compute a volume_path for them. +func resolveResourceReference(root dyn.Value, s string) string { + p, ok := dynvar.PureReferenceToPath(s) + if !ok || p[0].Key() != "resources" { + return s + } + rv, err := dyn.GetByPath(root, p) + if err != nil { + return s + } + rs, ok := rv.AsString() + if !ok { + return s + } + return rs +} diff --git a/bundle/config/mutator/initialize_volume_paths_test.go b/bundle/config/mutator/initialize_volume_paths_test.go new file mode 100644 index 00000000000..c2816d0642e --- /dev/null +++ b/bundle/config/mutator/initialize_volume_paths_test.go @@ -0,0 +1,163 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/require" +) + +func TestInitializeVolumePaths(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Schemas: map[string]*resources.Schema{ + "my": { + CreateSchema: catalog.CreateSchema{ + CatalogName: "main", + Name: "myschema", + }, + }, + }, + Volumes: map[string]*resources.Volume{ + "bar": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volbar", + }, + }, + // foo references the schema's name; InitializeVolumePaths resolves + // it locally to compute the path without rewriting schema_name. + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.my.name}", + Name: "volfoo", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.NoError(t, diags.Error()) + + require.Equal(t, "/Volumes/main/myschema/volbar", b.Config.Resources.Volumes["bar"].VolumePath) + + foo := b.Config.Resources.Volumes["foo"] + require.Equal(t, "/Volumes/main/myschema/volfoo", foo.VolumePath) + // The schema_name reference must be preserved, not replaced with the resolved value. + require.Equal(t, "${resources.schemas.my.name}", foo.SchemaName) +} + +func TestInitializeVolumePaths_UnresolvedReference(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + // The reference cannot be resolved locally, so it is embedded into + // the path verbatim to be resolved later during plan or deploy. + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.missing.name}", + Name: "volfoo", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.NoError(t, diags.Error()) + require.Equal(t, "/Volumes/main/${resources.schemas.missing.name}/volfoo", b.Config.Resources.Volumes["foo"].VolumePath) + // The schema_name reference itself must be preserved, not rewritten. + require.Equal(t, "${resources.schemas.missing.name}", b.Config.Resources.Volumes["foo"].SchemaName) +} + +func TestInitializeVolumePaths_RejectsUserProvidedPath(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volfoo", + }, + VolumePath: "/Volumes/bogus/path/set-by-user", + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.ErrorContains(t, diags.Error(), "volume_path is computed and read-only") +} + +func TestInitializeVolumePaths_MalformedReference(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + // A malformed reference must not leak into the computed path. + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "${resources.volumes.bar.bad..syntax}", + SchemaName: "myschema", + Name: "volfoo", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, InitializeVolumePaths()) + require.NoError(t, diags.Error()) + require.Empty(t, b.Config.Resources.Volumes["foo"].VolumePath) +} + +func TestVolumePathPipeline_ResolvesCrossVolumeReference(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "bar": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volbar", + }, + }, + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "volfoo", + Comment: "${resources.volumes.bar.volume_path}", + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq( + t.Context(), + b, + InitializeVolumePaths(), + ResolveVolumePathReferencesOnlyResources(), + ) + require.NoError(t, diags.Error()) + require.Equal(t, "/Volumes/main/myschema/volbar", b.Config.Resources.Volumes["bar"].VolumePath) + require.Equal(t, "/Volumes/main/myschema/volbar", b.Config.Resources.Volumes["foo"].Comment) +} diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 0aa73b575dd..a79810feb8a 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -50,6 +50,7 @@ type resolveVariableReferences struct { prefixes []string pattern dyn.Pattern lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error) + allowPathFn func(dyn.Path) bool extraRounds int // includeResources allows resolving variables in 'resources', otherwise, they are excluded. @@ -117,6 +118,34 @@ func ResolveVariableReferencesInLookup() bundle.Mutator { } } +// ResolveVolumePathReferencesOnlyResources resolves only references to resources.volumes.*.volume_path. +func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { + return &resolveVariableReferences{ + prefixes: []string{"resources"}, + lookupFn: lookupVolumePath, + allowPathFn: isVolumePathReferencePath, + extraRounds: maxResolutionRounds - 1, + includeResources: true, + } +} + +// lookupVolumePath resolves a reference to resources.volumes..volume_path. +// +// A volume's volume_path is empty only when it could not be computed, for example because +// catalog_name, schema_name, or name is unset or contains a malformed reference (see +// Volume.ComputeVolumePath). Resolving such a reference to "" would silently inject an empty +// string into the referrer, so we return an actionable error instead. +func lookupVolumePath(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { + result, err := lookup(v, path, b) + if err != nil { + return dyn.InvalidValue, err + } + if s, ok := result.AsString(); ok && s == "" { + return dyn.InvalidValue, fmt.Errorf("cannot resolve ${%s}: volume_path could not be computed; set catalog_name, schema_name, and name and ensure they contain no malformed references", path.String()) + } + return result, nil +} + func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { if path.String() == "workspace.file_path" { @@ -252,6 +281,9 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn // Perform resolution only if the path starts with one of the specified prefixes. if slices.ContainsFunc(prefixes, path.HasPrefix) { + if m.allowPathFn != nil && !m.allowPathFn(path) { + return dyn.InvalidValue, dynvar.ErrSkipResolution + } if slices.Contains(m.excludePaths, path.String()) { return dyn.InvalidValue, dynvar.ErrSkipResolution } @@ -334,3 +366,12 @@ func getAllKeys(root dyn.Value) ([]string, error) { return keys, nil } + +func isVolumePathReferencePath(path dyn.Path) bool { + if len(path) != 4 { + return false + } + return path[0].Key() == "resources" && + path[1].Key() == "volumes" && + path[3].Key() == "volume_path" +} diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 876980e9486..d8e7ca931db 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/require" ) @@ -63,3 +64,69 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { testCase.assert(t, b) } } + +func TestResolveVolumePathReferencesOnlyResources(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + VolumePath: "/Volumes/main/myschema/myvolume", + }, + "volumePathRef": {}, + "otherRef": {}, + }, + }, + }, + } + + b.Config.Resources.Volumes["volumePathRef"].Comment = "${resources.volumes.foo.volume_path}" + b.Config.Resources.Volumes["otherRef"].Comment = "${resources.volumes.foo.name}" + + diags := bundle.Apply(t.Context(), b, ResolveVolumePathReferencesOnlyResources()) + require.NoError(t, diags.Error()) + require.Equal(t, "/Volumes/main/myschema/myvolume", b.Config.Resources.Volumes["volumePathRef"].Comment) + require.Equal(t, "${resources.volumes.foo.name}", b.Config.Resources.Volumes["otherRef"].Comment) +} + +func TestResolveVolumePathReferencesOnlyResources_MissingTarget(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": {}, + }, + }, + }, + } + b.Config.Resources.Volumes["foo"].Comment = "${resources.volumes.missing.volume_path}" + + diags := bundle.Apply(t.Context(), b, ResolveVolumePathReferencesOnlyResources()) + require.ErrorContains(t, diags.Error(), "reference does not exist: ${resources.volumes.missing.volume_path}") +} + +func TestResolveVolumePathReferencesOnlyResources_UncomputedTargetErrors(t *testing.T) { + // When the target volume exists but its volume_path could not be computed (for + // example because catalog_name/schema_name/name is unset), the reference must + // error rather than silently resolving to an empty string. + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": {}, + "ref": {}, + }, + }, + }, + } + b.Config.Resources.Volumes["ref"].Comment = "${resources.volumes.foo.volume_path}" + + diags := bundle.Apply(t.Context(), b, ResolveVolumePathReferencesOnlyResources()) + require.ErrorContains(t, diags.Error(), "volume_path could not be computed") +} + +func TestIsVolumePathReferencePath(t *testing.T) { + require.True(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.volume_path"))) + require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.name"))) + require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.jobs.foo.name"))) +} diff --git a/bundle/config/resources/volume.go b/bundle/config/resources/volume.go index 4b1faecc9c0..9e97b5c3a27 100644 --- a/bundle/config/resources/volume.go +++ b/bundle/config/resources/volume.go @@ -2,10 +2,13 @@ package resources import ( "context" + "fmt" "net/url" + "strings" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" @@ -17,6 +20,9 @@ type Volume struct { BaseResource catalog.CreateVolumeRequestContent + // VolumePath is /Volumes/{catalog}/{schema}/{name}. Populated during initialize; not user-configurable. + VolumePath string `json:"volume_path,omitempty" bundle:"readonly"` + // List of grants to apply on this volume. Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } @@ -69,3 +75,22 @@ func (v *Volume) GetURL() string { func (v *Volume) GetName() string { return v.Name } + +// ComputeVolumePath returns the Unity Catalog volume path /Volumes/{catalog}/{schema}/{name}. +// +// A component that is still a pure ${...} reference (for example a remote field only +// known at plan or deploy time) is embedded verbatim, so the reference is carried into +// the path and resolved later by the normal interpolation passes. A component that +// contains "${" but is not a well-formed reference (malformed or partial) is rejected +// to keep it from leaking into the path, in which case the empty string is returned. +func (v *Volume) ComputeVolumePath() string { + for _, component := range []string{v.CatalogName, v.SchemaName, v.Name} { + if component == "" { + return "" + } + if strings.Contains(component, "${") && !dynvar.IsPureVariableReference(component) { + return "" + } + } + return fmt.Sprintf("/Volumes/%s/%s/%s", v.CatalogName, v.SchemaName, v.Name) +} diff --git a/bundle/config/resources/volume_test.go b/bundle/config/resources/volume_test.go index ef44a2e61de..cb14a19ed45 100644 --- a/bundle/config/resources/volume_test.go +++ b/bundle/config/resources/volume_test.go @@ -5,10 +5,56 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) +func TestComputeVolumePath(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "myvol", + }, + } + require.Equal(t, "/Volumes/main/myschema/myvol", v.ComputeVolumePath()) +} + +func TestComputeVolumePath_PureReferenceEmbedded(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.my.name}", + Name: "myvol", + }, + } + // A pure reference is embedded verbatim so it can be resolved later (plan/deploy). + require.Equal(t, "/Volumes/main/${resources.schemas.my.name}/myvol", v.ComputeVolumePath()) +} + +func TestComputeVolumePath_MalformedReference(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "${resources.schemas.my.bad..syntax}", + Name: "myvol", + }, + } + // A malformed reference must not leak into the path. + require.Empty(t, v.ComputeVolumePath()) +} + +func TestComputeVolumePath_MissingField(t *testing.T) { + v := &Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + Name: "myvol", + }, + } + require.Empty(t, v.ComputeVolumePath()) +} + func TestVolumeNotFound(t *testing.T) { ctx := t.Context() diff --git a/bundle/configsync/defaults.go b/bundle/configsync/defaults.go index cc56b361aff..9e080edeae0 100644 --- a/bundle/configsync/defaults.go +++ b/bundle/configsync/defaults.go @@ -100,6 +100,7 @@ var serverSideDefaults = map[string]any{ // Volume fields "resources.volumes.*.storage_location": alwaysSkip, + "resources.volumes.*.volume_path": alwaysSkip, // SQL warehouse fields "resources.sql_warehouses.*.creator_name": alwaysSkip, diff --git a/bundle/deploy/terraform/tfdyn/convert_volume.go b/bundle/deploy/terraform/tfdyn/convert_volume.go index 287ddee0c6b..e8becaea175 100644 --- a/bundle/deploy/terraform/tfdyn/convert_volume.go +++ b/bundle/deploy/terraform/tfdyn/convert_volume.go @@ -11,6 +11,12 @@ import ( ) func convertVolumeResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // volume_path is computed by the API and cannot be configured in Terraform. + vin, err := dyn.DropKeys(vin, []string{"volume_path"}) + if err != nil { + return dyn.InvalidValue, err + } + // Normalize the output value to the target schema. vout, diags := convert.Normalize(schema.ResourceVolume{}, vin) for _, diag := range diags { diff --git a/bundle/deploy/terraform/tfdyn/convert_volume_test.go b/bundle/deploy/terraform/tfdyn/convert_volume_test.go index bd8798780ad..f21b1024483 100644 --- a/bundle/deploy/terraform/tfdyn/convert_volume_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_volume_test.go @@ -22,6 +22,7 @@ func TestConvertVolume(t *testing.T) { StorageLocation: "s3://bucket/path", VolumeType: "EXTERNAL", }, + VolumePath: "/Volumes/catalog/schema/name", Grants: []catalog.PrivilegeAssignment{ { Privileges: []catalog.Privilege{catalog.PrivilegeReadVolume}, diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 6db00b681fc..43aeaa89c41 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -952,7 +952,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // This means input and state must be compatible: input can have more fields, but existing fields should not be moved // This means one cannot refer to fields not present in state (e.g. ${resources.jobs.foo.permissions}) - refs, err := extractReferences(configRoot.Value(), node) + refs, err := extractReferences(configRoot.Value(), node, adapter.StateType()) if err != nil { return nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) } @@ -1041,13 +1041,15 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } -// ExtractReferences extracts all variable references from the config subtree rooted at node. +// ExtractReferences extracts variable references from the config subtree rooted at node, +// keeping only those whose field path exists in stateType (references in input-only or +// bundle:"readonly" fields, such as volumes' computed volume_path, are skipped). // Returns a map from structpath string (field path within the resource) to template string. -func ExtractReferences(root dyn.Value, node string) (map[string]string, error) { - return extractReferences(root, node) +func ExtractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { + return extractReferences(root, node, stateType) } -func extractReferences(root dyn.Value, node string) (map[string]string, error) { +func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { nodeType := config.GetResourceTypeFromKey(node) refs := make(map[string]string) @@ -1075,11 +1077,23 @@ func extractReferences(root dyn.Value, node string) (map[string]string, error) { if !ok { return nil } - // Store the original string that contains references, not individual references. - // Convert dyn.Path to structpath string because refs are later parsed by structpath.ParsePath. - // dyn.Path.String() uses dot notation which is ambiguous for keys containing dots; - // structpath uses bracket notation (['key.with.dots']) which round-trips correctly. - refs[dynPathToStructPath(p).String()] = ref.Str + // Convert dyn.Path to structpath: dyn.Path.String() uses dot notation + // which is ambiguous for keys containing dots; structpath uses bracket + // notation (['key.with.dots']) which round-trips correctly. + fieldPath := dynPathToStructPath(p) + + // References are resolved against the resource's state type, not its input + // config (see the note in PlanResources and dresources.TestInputSubset). A + // field that exists in input but not in state — most notably a + // bundle:"readonly" field such as volumes' computed volume_path — is dropped + // from state before deploy, so a reference it carries cannot be resolved into + // the state and must not be treated as a dependency here. Such references are + // still made available to other resources that read the field (for example + // ${resources.volumes.x.volume_path}) earlier during initialize. + if structaccess.ValidatePath(stateType, fieldPath) == nil { + // Store the original string that contains references, not individual references. + refs[fieldPath.String()] = ref.Str + } return nil }) if err != nil { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index c7f6e8dbf36..a098da7561c 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -1,11 +1,13 @@ package direct import ( + "bytes" "testing" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,6 +42,38 @@ func TestDynPathToStructPath(t *testing.T) { } } +// extractReferences gates references on the resource's state type, so a reference +// stored in a field that exists in the input config but not in state (most notably a +// bundle:"readonly" field such as volumes' computed volume_path) must not become a +// dependency. References in state fields (e.g. comment) are still extracted. +func TestExtractReferences_ExcludesReadonlyFields(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + // The volume state type is catalog.CreateVolumeRequestContent, which has + // comment but not volume_path. + stateType := adapters["volumes"].StateType() + + const yml = ` +resources: + volumes: + v: + catalog_name: main + schema_name: myschema + name: myvol + comment: "${resources.schemas.kept.name}" + volume_path: "/Volumes/main/${resources.schemas.dropped.name}/myvol" +` + root, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yml)) + require.NoError(t, err) + + refs, err := extractReferences(root, "resources.volumes.v", stateType) + require.NoError(t, err) + + assert.Equal(t, map[string]string{ + "comment": "${resources.schemas.kept.name}", + }, refs) +} + func TestShouldSkipBackendDefault_ManagedPropertiesOnly(t *testing.T) { // Rules mirror the schemas backend_defaults in resources.yml, but the test is // deliberately self-contained so that edits to resources.yml don't break it. diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 315d1791666..5dc5b3d3425 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -114,7 +114,7 @@ func BuildStateFromTF( return fmt.Errorf("%s: PrepareState: %w", node, err) } - refs, err := direct.ExtractReferences(configRoot.Value(), node) + refs, err := direct.ExtractReferences(configRoot.Value(), node, adapter.StateType()) if err != nil { return fmt.Errorf("%s: extracting references: %w", node, err) } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8f54c141f80..f6763e05502 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -148,6 +148,13 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // After PythonMutator, mutators must not change bundle resources, or such changes are not // going to be visible in Python code. + // Compute resources.volumes.*.volume_path and resolve references to it. This must run + // after PythonMutator: volume_path is a computed, read-only field that the PyDABs Volume + // model does not declare, so exposing it to Python would fail resource loading. Like the + // "deployment" metadata below, it is intentionally not visible to Python code. + mutator.InitializeVolumePaths(), + mutator.ResolveVolumePathReferencesOnlyResources(), + // Resolve --select selectors against the materialized resources: normalize // each to its "type.name" form and validate it exists. Runs after all resource // mutations so that dynamically added resources are visible. This does not diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index dd974dd626e..2dedfa5ca8a 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -31,7 +31,6 @@ package terraform_dabs_map // schemas / databricks_schema: 1 tf-only // secret_scopes / databricks_secret_scope: 1 tf-only // sql_warehouses / databricks_sql_endpoint: 2 tf-only -// volumes / databricks_volume: 1 tf-only // TerraformToDABsFieldMap maps DABs group name → nested TF segments → DABs segment name. // Navigate using TF field name segments; DABs is the corresponding DABs name when it differs. @@ -590,9 +589,6 @@ var TerraformOnlyFields = map[string]FieldSet{ "data_source_id": {}, "no_wait": {}, }, - "volumes": { - "volume_path": {}, - }, } // DABsToTerraformRenameMap maps DABs group name → nested DABs segments → TF segment name.