Expose computed volume_path during initialize

NEXT_CHANGELOG.md

@@ -7,6 +7,9 @@
 ### CLI
 
 ### Bundles
+* Expose a computed, read-only `volume_path` on `resources.volumes.*` so configs can reference a volume's Unity Catalog path via `${resources.volumes.<key>.volume_path}` instead of hardcoding `/Volumes/<catalog>/<schema>/<name>` ([#5550](https://github.com/databricks/cli/pull/5550)).
+  * `volume_path` is derived purely from the volume's `catalog_name`, `schema_name`, and `name`, so the reference is resolved early (at initialize) and inlined into the referring field. Referencing `volume_path` therefore does not make the referring resource depend on the volume during deploy; if `catalog_name`/`schema_name`/`name` themselves reference other resources, the referrer depends on those resources instead.
+  * Supported on the direct deployment engine (`DATABRICKS_BUNDLE_ENGINE=direct`). On the Terraform engine `volume_path` is dropped before apply, and a `volume_path` whose components embed a value only known after deploy (for example `${resources.jobs.<key>.creator_user_name}`) is not supported.
 
 ### Dependency updates
 

acceptance/bundle/invariant/configs/volume_path_job_ref.yml.tmpl

@@ -0,0 +1,24 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# A job parameter references a volume's computed volume_path, and the volume's
+# schema_name/name are references to another job's remote creator_user_name. So
+# volume_path embeds a reference at initialize that only resolves at deploy,
+# exercising the deferred-resolution chain (foo -> data.volume_path -> process)
+# end to end with no drift afterwards.
+resources:
+  jobs:
+    foo:
+      name: test-job-foo-$UNIQUE_NAME
+
+    process:
+      name: test-job-process-$UNIQUE_NAME
+      parameters:
+        - name: data_path
+          default: ${resources.volumes.data.volume_path}
+
+  volumes:
+    data:
+      catalog_name: main
+      schema_name: ${resources.jobs.foo.creator_user_name}
+      name: ${resources.jobs.foo.creator_user_name}

acceptance/bundle/invariant/continue_293/test.toml

@@ -15,6 +15,10 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 # Dotted pipeline configuration keys are not supported on v0.293.0
 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"]
 
+# Computed volume_path is not supported on v0.293.0, so the seed deploy cannot
+# resolve ${resources.volumes.*.volume_path}. Covered by no_drift on direct.
+EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"]
+
 # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min
 # per variant (two full deploys at 1000 tasks) without incremental coverage.
 EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]

acceptance/bundle/invariant/migrate/test.toml

@@ -30,3 +30,10 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml
 # migrate deploys via Terraform first, and the TF provider rejects an uppercase
 # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct.
 EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"]
+
+# volume_path embeds ${resources.jobs.foo.creator_user_name}; the terraform
+# interpolator converts it to ${databricks_job.foo.creator_user_name}, but the
+# terraform databricks_job resource does not export creator_user_name, so the
+# terraform seed deploy fails. Same class as no_cross_resource_ref above.
+# Covered by no_drift on direct.
+EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"]

acceptance/bundle/invariant/test.toml

@@ -67,6 +67,7 @@ EnvMatrix.INPUT_CONFIG = [
     "vector_search_index.yml.tmpl",
     "volume.yml.tmpl",
     "volume_external.yml.tmpl",
+    "volume_path_job_ref.yml.tmpl",
     "volume_uppercase_name.yml.tmpl",
 ]
 

acceptance/bundle/python/volumes-support/output.txt

@@ -17,12 +17,14 @@
         "catalog_name": "my_catalog",
         "name": "My Volume (updated)",
         "schema_name": "my_schema",
+        "volume_path": "/Volumes/my_catalog/my_schema/My Volume (updated)",
         "volume_type": "MANAGED"
       },
       "my_volume_2": {
         "catalog_name": "my_catalog_2",
         "name": "My Volume (2) (updated)",
         "schema_name": "my_schema_2",
+        "volume_path": "/Volumes/my_catalog_2/my_schema_2/My Volume (2) (updated)",
         "volume_type": "MANAGED"
       }
     }

acceptance/bundle/refschema/out.fields.txt

@@ -3476,6 +3476,7 @@ resources.volumes.*.updated_at	int64	REMOTE
 resources.volumes.*.updated_by	string	REMOTE
 resources.volumes.*.url	string	INPUT
 resources.volumes.*.volume_id	string	REMOTE
+resources.volumes.*.volume_path	string	INPUT
 resources.volumes.*.volume_type	catalog.VolumeType	ALL
 resources.volumes.*.grants.full_name	string	ALL
 resources.volumes.*.grants.securable_type	string	ALL

acceptance/bundle/resource_deps/bad_syntax/output.txt

@@ -6,6 +6,7 @@
       "catalog_name": "mycatalog",
       "name": "barname",
       "schema_name": "myschema",
+      "volume_path": "/Volumes/mycatalog/myschema/barname",
       "volume_type": "MANAGED"
     },
     "foo": {

acceptance/bundle/resource_deps/computed_volume_path/databricks.yml.tmpl

@@ -0,0 +1,20 @@
+bundle:
+  name: testbundle-${UNIQUE_NAME}
+
+resources:
+  schemas:
+    my:
+      catalog_name: main
+      name: myschema-${UNIQUE_NAME}
+
+  volumes:
+    bar:
+      catalog_name: main
+      schema_name: myschema-${UNIQUE_NAME}
+      name: volumebar-${UNIQUE_NAME}
+
+    foo:
+      catalog_name: main
+      schema_name: myschema-${UNIQUE_NAME}
+      name: volumefoo-${UNIQUE_NAME}
+      comment: ${resources.volumes.bar.volume_path}

acceptance/bundle/resource_deps/computed_volume_path/output.txt

@@ -0,0 +1,36 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "schemas": {
+    "my": {
+      "catalog_name": "main",
+      "name": "myschema-[UNIQUE_NAME]"
+    }
+  },
+  "volumes": {
+    "bar": {
+      "catalog_name": "main",
+      "name": "volumebar-[UNIQUE_NAME]",
+      "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]",
+      "volume_type": "MANAGED"
+    },
+    "foo": {
+      "catalog_name": "main",
+      "comment": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]",
+      "name": "volumefoo-[UNIQUE_NAME]",
+      "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]",
+      "volume_type": "MANAGED"
+    }
+  }
+}
+
+>>> [CLI] bundle plan
+create schemas.my
+create volumes.bar
+create volumes.foo
+
+Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> print_requests.py --get //unity

acceptance/bundle/resource_deps/computed_volume_path/script

@@ -0,0 +1,4 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+trace $CLI bundle validate -o json | jq .resources
+trace $CLI bundle plan
+trace print_requests.py --get //unity

acceptance/bundle/resource_deps/implicit_deps_volume/output.txt

@@ -17,6 +17,7 @@
       "catalog_name": "${resources.catalogs.my_catalog.name}",
       "name": "myvolume",
       "schema_name": "${resources.schemas.my_schema.name}",
+      "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myvolume",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/non_existent_field/output.txt

@@ -6,12 +6,14 @@
       "catalog_name": "mycatalog",
       "name": "barname",
       "schema_name": "myschema",
+      "volume_path": "/Volumes/mycatalog/myschema/barname",
       "volume_type": "MANAGED"
     },
     "foo": {
       "catalog_name": "${resources.volumes.bar.non_existent}",
       "name": "myname",
       "schema_name": "${resources.volumes.bar.schema_name}",
+      "volume_path": "/Volumes/${resources.volumes.bar.non_existent}/myschema/myname",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/remote_field_storage_location/output.txt

@@ -12,13 +12,15 @@
       "catalog_name": "main",
       "name": "volumebar-[UNIQUE_NAME]",
       "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]",
       "volume_type": "MANAGED"
     },
     "foo": {
       "catalog_name": "main",
       "comment": "${resources.volumes.bar.storage_location}",
       "name": "volumefoo-[UNIQUE_NAME]",
       "schema_name": "${resources.schemas.my.name}",
+      "volume_path": "/Volumes/main/myschema-[UNIQUE_NAME]/volumefoo-[UNIQUE_NAME]",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/resources_var/output.txt

@@ -21,12 +21,14 @@
       "catalog_name": "mycatalog",
       "name": "barname",
       "schema_name": "myschema",
+      "volume_path": "/Volumes/mycatalog/myschema/barname",
       "volume_type": "MANAGED"
     },
     "foo": {
       "catalog_name": "${resources.volumes.bar.catalog_name}",
       "name": "myname",
       "schema_name": "${resources.volumes.bar.schema_name}",
+      "volume_path": "/Volumes/mycatalog/myschema/myname",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/output.txt

@@ -27,6 +27,7 @@
       "catalog_name": "${resources.schemas.bar.catalog_name}",
       "name": "myname",
       "schema_name": "${resources.schemas.bar.name}",
+      "volume_path": "/Volumes/mycatalog/dev_[USERNAME]_myschema/myname",
       "volume_type": "MANAGED"
     }
   }

acceptance/bundle/resource_deps/volume_path_contains_id/databricks.yml

@@ -0,0 +1,29 @@
+bundle:
+  name: test-bundle
+
+# volume_path is a CLI-only computed field (not API state). When a component
+# (catalog_name, schema_name, name) is an unresolved ${...} reference, it stays
+# embedded in volume_path so it resolves later at plan/deploy.
+#
+# Chain under test: bar.name is a remote value (baz.id), so bar.volume_path is
+# computed at initialize with that reference intact; foo.comment picks up that
+# string and resolves at deploy to the fully-resolved path.
+#
+# baz.id is used instead of baz.storage_location because the resolved value
+# becomes bar's name, and the testserver can't route a name containing the
+# slashes of a storage_location URL.
+resources:
+  volumes:
+    baz:
+      catalog_name: mycatalog
+      schema_name: myschema
+      name: bazname
+    bar:
+      catalog_name: mycatalog
+      schema_name: myschema
+      name: ${resources.volumes.baz.id}
+    foo:
+      catalog_name: mycatalog
+      schema_name: myschema
+      name: fooname
+      comment: ${resources.volumes.bar.volume_path}

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.direct.txt

@@ -0,0 +1,6 @@
+
+>>> errcode [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.direct.json

@@ -0,0 +1,31 @@
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname",
+    "name": "fooname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "mycatalog.myschema.bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.requests.terraform.json

@@ -0,0 +1,43 @@
+{
+  "method": "GET",
+  "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.bazname"
+}
+{
+  "method": "GET",
+  "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.fooname"
+}
+{
+  "method": "GET",
+  "path": "/api/2.1/unity-catalog/volumes/mycatalog.myschema.mycatalog.myschema.bazname"
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "comment": "/Volumes/mycatalog/myschema/mycatalog.myschema.bazname",
+    "name": "fooname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/volumes",
+  "body": {
+    "catalog_name": "mycatalog",
+    "name": "mycatalog.myschema.bazname",
+    "schema_name": "myschema",
+    "volume_type": "MANAGED"
+  }
+}

acceptance/bundle/resource_deps/volume_path_contains_id/out.deploy.terraform.txt

@@ -0,0 +1,6 @@
+
+>>> errcode [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!

acceptance/bundle/resource_deps/volume_path_contains_id/out.destroy.direct.txt

@@ -0,0 +1,17 @@
+The following resources will be deleted:
+  delete resources.volumes.bar
+  delete resources.volumes.baz
+  delete resources.volumes.foo
+
+This action will result in the deletion of the following volumes.
+For managed volumes, the files stored in the volume are also deleted from your
+cloud tenant within 30 days. For external volumes, the metadata about the volume
+is removed from the catalog, but the underlying files are not deleted:
+  delete resources.volumes.bar
+  delete resources.volumes.baz
+  delete resources.volumes.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Deleting files...
+Destroy complete!