Add preliminary job_runs resource

NEXT_CHANGELOG.md

@@ -7,6 +7,7 @@
 ### CLI
 
 ### Bundles
+* Add a `job_runs` resource (direct engine only) that triggers a job run as part of `bundle deploy`. Any change to the run configuration re-triggers the run; otherwise it is considered up to date ([#5603](https://github.com/databricks/cli/pull/5603)).
 
 ### Dependency updates
 

acceptance/bundle/invariant/configs/job_run.yml.tmpl

@@ -0,0 +1,20 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: test-job-$UNIQUE_NAME
+      tasks:
+        - task_key: only_task
+          notebook_task:
+            notebook_path: /Shared/notebook
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            instance_pool_id: $TEST_INSTANCE_POOL_ID
+            num_workers: 1
+
+  job_runs:
+    foo_run:
+      job_id: ${resources.jobs.foo.id}

acceptance/bundle/invariant/continue_293/test.toml

@@ -12,6 +12,9 @@ EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoi
 # genie_spaces resource is not supported on v0.293.0
 EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 
+# job_runs resource is not supported on v0.293.0
+EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"]
+
 # Dotted pipeline configuration keys are not supported on v0.293.0
 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"]
 

acceptance/bundle/invariant/migrate/test.toml

@@ -2,6 +2,10 @@
 EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"]
 EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml.tmpl"]
 
+# job_runs is a direct-only resource with no terraform converter, so the
+# terraform deploy that seeds the migration fails for it.
+EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"]
+
 # Error: Catalog resources are only supported with direct deployment mode
 EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"]
 EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"]

acceptance/bundle/invariant/test.toml

@@ -37,6 +37,7 @@ EnvMatrix.INPUT_CONFIG = [
     "job_pydabs_1000_tasks.yml.tmpl",
     "job_cross_resource_ref.yml.tmpl",
     "job_permission_ref.yml.tmpl",
+    "job_run.yml.tmpl",
     "job_run_job_ref.yml.tmpl",
     "job_with_depends_on.yml.tmpl",
     "job_with_permissions.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -756,6 +756,45 @@ resources.genie_spaces.*.permissions[*].group_name	string	ALL
 resources.genie_spaces.*.permissions[*].level	iam.PermissionLevel	ALL
 resources.genie_spaces.*.permissions[*].service_principal_name	string	ALL
 resources.genie_spaces.*.permissions[*].user_name	string	ALL
+resources.job_runs.*.dbt_commands	[]string	ALL
+resources.job_runs.*.dbt_commands[*]	string	ALL
+resources.job_runs.*.id	string	INPUT
+resources.job_runs.*.idempotency_token	string	ALL
+resources.job_runs.*.jar_params	[]string	ALL
+resources.job_runs.*.jar_params[*]	string	ALL
+resources.job_runs.*.job_id	int64	ALL
+resources.job_runs.*.job_parameters	map[string]string	ALL
+resources.job_runs.*.job_parameters.*	string	ALL
+resources.job_runs.*.lifecycle	resources.Lifecycle	INPUT
+resources.job_runs.*.lifecycle.prevent_destroy	bool	INPUT
+resources.job_runs.*.modified_status	string	INPUT
+resources.job_runs.*.notebook_params	map[string]string	ALL
+resources.job_runs.*.notebook_params.*	string	ALL
+resources.job_runs.*.only	[]string	ALL
+resources.job_runs.*.only[*]	string	ALL
+resources.job_runs.*.performance_target	jobs.PerformanceTarget	ALL
+resources.job_runs.*.pipeline_params	*jobs.PipelineParams	ALL
+resources.job_runs.*.pipeline_params.full_refresh	bool	ALL
+resources.job_runs.*.pipeline_params.full_refresh_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.full_refresh_selection[*]	string	ALL
+resources.job_runs.*.pipeline_params.refresh_flow_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.refresh_flow_selection[*]	string	ALL
+resources.job_runs.*.pipeline_params.refresh_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.refresh_selection[*]	string	ALL
+resources.job_runs.*.pipeline_params.reset_checkpoint_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.reset_checkpoint_selection[*]	string	ALL
+resources.job_runs.*.python_named_params	map[string]string	ALL
+resources.job_runs.*.python_named_params.*	string	ALL
+resources.job_runs.*.python_params	[]string	ALL
+resources.job_runs.*.python_params[*]	string	ALL
+resources.job_runs.*.queue	*jobs.QueueSettings	ALL
+resources.job_runs.*.queue.enabled	bool	ALL
+resources.job_runs.*.run_id	int64	REMOTE
+resources.job_runs.*.spark_submit_params	[]string	ALL
+resources.job_runs.*.spark_submit_params[*]	string	ALL
+resources.job_runs.*.sql_params	map[string]string	ALL
+resources.job_runs.*.sql_params.*	string	ALL
+resources.job_runs.*.url	string	INPUT
 resources.jobs.*.budget_policy_id	string	ALL
 resources.jobs.*.continuous	*jobs.Continuous	ALL
 resources.jobs.*.continuous.pause_status	jobs.PauseStatus	ALL

acceptance/bundle/resources/job_runs/basic/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+  name: job-runs-basic
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}

acceptance/bundle/resources/job_runs/basic/output.txt

@@ -0,0 +1,90 @@
+
+=== deploy triggers exactly one run
+>>> [CLI] bundle validate
+Name: job-runs-basic
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+
+Validation OK!
+
+>>> [CLI] bundle plan
+create job_runs.my_run
+create jobs.my_job
+
+Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle summary
+Name: job-runs-basic
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+Resources:
+  Job Runs:
+    my_run:
+      Name: 
+      URL:  (not deployed)
+  Jobs:
+    my_job:
+      Name: my-job
+      URL:  (not deployed)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle summary
+Name: job-runs-basic
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+Resources:
+  Job Runs:
+    my_run:
+      Name: 
+      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID]
+  Jobs:
+    my_job:
+      Name: my-job
+      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID]
+
+=== exactly one run-now request was made
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "job_id": [MY_JOB_ID]
+  }
+}
+
+=== the triggered run id is stored in state
+>>> read_id.py my_run
+[MY_RUN_ID]
+
+>>> read_id.py my_job
+[MY_JOB_ID]
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/basic/script

@@ -0,0 +1,24 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm out.requests.txt
+}
+trap cleanup EXIT
+
+title "deploy triggers exactly one run"
+trace $CLI bundle validate
+trace $CLI bundle plan
+trace $CLI bundle summary
+trace $CLI bundle deploy
+
+# confirm that redeploy does not trigger a second run
+trace $CLI bundle deploy
+
+trace $CLI bundle plan
+trace $CLI bundle summary
+
+title "exactly one run-now request was made"
+trace print_requests.py //jobs/run-now
+
+title "the triggered run id is stored in state"
+trace read_id.py my_run
+trace read_id.py my_job

acceptance/bundle/resources/job_runs/basic/test.toml

@@ -0,0 +1,4 @@
+# job_runs is a direct-engine-only resource; the Terraform provider has no
+# equivalent, so restrict the matrix to direct.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = true

acceptance/bundle/resources/job_runs/job_parameters/databricks.yml

@@ -0,0 +1,26 @@
+bundle:
+  name: job-runs-job-parameters
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      parameters:
+        - name: env
+          default: dev
+        - name: region
+          default: us
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}
+      # Override only one of the job's two parameters. GetRun resolves the full
+      # parameter set (including the un-overridden `region` default), so without
+      # job_parameters in ignore_remote_changes this drift would recreate the run
+      # on every plan.
+      job_parameters:
+        env: prod

acceptance/bundle/resources/job_runs/job_parameters/output.txt

@@ -0,0 +1,49 @@
+
+=== deploy triggers the run with only the overridden parameter
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "job_id": [NUMID],
+    "job_parameters": {
+      "env": "prod"
+    }
+  }
+}
+
+=== plan is stable: the resolved job default does not look like drift
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle summary
+Name: job-runs-job-parameters
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default
+Resources:
+  Job Runs:
+    my_run:
+      Name: 
+      URL:  [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?w=[NUMID]
+  Jobs:
+    my_job:
+      Name: my-job
+      URL:  [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID]
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/job_parameters/script

@@ -0,0 +1,13 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm out.requests.txt
+}
+trap cleanup EXIT
+
+title "deploy triggers the run with only the overridden parameter"
+trace $CLI bundle deploy
+trace print_requests.py //jobs/run-now
+
+title "plan is stable: the resolved job default does not look like drift"
+trace $CLI bundle plan
+trace $CLI bundle summary

acceptance/bundle/resources/job_runs/job_parameters/test.toml

@@ -0,0 +1,4 @@
+# job_runs is a direct-engine-only resource; the Terraform provider has no
+# equivalent, so restrict the matrix to direct.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = true

acceptance/bundle/resources/job_runs/redeploy/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: job-runs-redeploy
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}
+      job_parameters:
+        env: dev