Make --profile take precedence over auth environment variables

NEXT_CHANGELOG.md

@@ -8,6 +8,7 @@
 * `workspace export-dir` no longer aborts when a workspace object's name is not a legal local filename (e.g. a notebook named `New Notebook 2026-05-04 13:54:24` whose `:` is illegal on Windows). Such files are now exported under a sanitized name with a warning and the export completes ([#5171](https://github.com/databricks/cli/issues/5171)).
 * `ssh connect` now opens an interactive `bash` login shell by default instead of the compute image's default `/bin/sh`, falling back gracefully when `bash` is unavailable. Passing an explicit remote command (`-- <cmd>`) is unaffected ([#5687](https://github.com/databricks/cli/pull/5687)).
 * `ssh connect` interactive sessions now start in the user's workspace home folder (`/Workspace/Users/<email>`) instead of the OS home directory, falling back to the OS home when that folder is unavailable ([#5688](https://github.com/databricks/cli/pull/5688)).
+* An explicitly selected profile (the `--profile` flag or a bundle's `workspace.profile`) now takes precedence over authentication environment variables (`DATABRICKS_HOST`, `DATABRICKS_TOKEN`, etc.). Previously these env vars silently shadowed the selected profile's host and credentials. Environment variables still fill auth fields the profile leaves empty (e.g. a host-only profile combined with `DATABRICKS_TOKEN`). A profile picked up from `DATABRICKS_CONFIG_PROFILE` keeps the SDK's default env-first precedence ([#5096](https://github.com/databricks/cli/issues/5096)).
 
 ### Bundles
 * Add documentation for the common bundle resource fields `permissions`, `lifecycle`, and `grants` in the JSON schema, so they surface in editor completions and the docs.

acceptance/cmd/auth/describe/profile-overrides-env/output.txt

@@ -0,0 +1,36 @@
+
+=== Describe with --profile overrides auth env vars (#5096)
+
+>>> [CLI] auth describe --profile my-workspace
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ profile: my-workspace (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server
+
+=== Describe with a host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] auth describe --profile host-only
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+  ✓ profile: host-only (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server

acceptance/cmd/auth/describe/profile-overrides-env/script

@@ -0,0 +1,26 @@
+sethome "./home"
+
+# A profile carries full credentials; a second profile carries only a host.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host  = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars pointing at a different (dev) workspace. Before the
+# fix for #5096 these shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "Describe with --profile overrides auth env vars (#5096)\n"
+trace $CLI auth describe --profile my-workspace
+
+# A host-only profile relies on the environment for the credential: the profile
+# must win for the host, but the env must still fill the token it omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "Describe with a host-only --profile fills the token from the environment (#5096)\n"
+trace $CLI auth describe --profile host-only

acceptance/cmd/auth/describe/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home"
+]

bundle/config/workspace.go

@@ -176,9 +176,24 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er
 
 	cfg := w.Config(ctx)
 
-	// If only the host is configured, we try and unambiguously match it to
-	// a profile in the user's databrickscfg file. Override the default loaders.
-	if w.Host != "" && w.Profile == "" {
+	switch {
+	case w.Profile != "":
+		// An explicit profile (from --profile or workspace.profile) must take
+		// precedence over authentication environment variables, mirroring
+		// MustWorkspaceClient. The SDK's default loader reads the environment
+		// before the config file and never overwrites an already-set field, so
+		// without this DATABRICKS_HOST/DATABRICKS_TOKEN would shadow the
+		// selected profile (issue #5096). See databrickscfg.ProfileAuthLoaders
+		// for the loader order and rationale.
+		//
+		// This also covers a bundle that sets both host and profile: the
+		// profile now wins for auth (previously the env did), and the existing
+		// ValidateConfigAndProfileHost check below still enforces that the
+		// bundle host matches the profile host.
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+	case w.Host != "":
+		// If only the host is configured, we try and unambiguously match it to
+		// a profile in the user's databrickscfg file. Override the default loaders.
 		cfg.Loaders = []config.Loader{
 			// Load auth creds from env vars
 			config.ConfigAttributes,

bundle/config/workspace_test.go

@@ -155,6 +155,54 @@ func TestWorkspaceClientNormalizesHostBeforeProfileResolution(t *testing.T) {
 	assert.Equal(t, "ws2", client.Config.Profile)
 }
 
+func TestWorkspaceClientProfileOverridesAuthEnv(t *testing.T) {
+	// An explicit profile (from --profile or workspace.profile) must win over
+	// authentication environment variables, mirroring MustWorkspaceClient.
+	// See https://github.com/databricks/cli/issues/5096.
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "tst",
+		Host:    "https://tst.cloud.databricks.test",
+		Token:   "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{Profile: "tst"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
+func TestWorkspaceClientProfileFillsAuthFromEnv(t *testing.T) {
+	// A host-only profile relies on the environment for credentials. The profile
+	// must take precedence for the host, but the env must still fill the token
+	// the profile does not provide (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "host-only",
+		Host:    "https://tst.cloud.databricks.test",
+	})
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_TOKEN", "env-token")
+
+	w := Workspace{Profile: "host-only"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "host-only", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	// The token is not in the profile, so it is filled from the environment.
+	assert.Equal(t, "env-token", client.Config.Token)
+}
+
 func TestWorkspaceConfigHTTPTimeout(t *testing.T) {
 	for _, tc := range []struct {
 		envVal string

cmd/root/auth.go

@@ -195,15 +195,23 @@ func MustAnyClient(cmd *cobra.Command, args []string) (bool, error) {
 
 func MustAccountClient(cmd *cobra.Command, args []string) error {
 	cfg := &config.Config{}
+	ctx := cmd.Context()
 
 	// The command-line profile flag takes precedence over DATABRICKS_CONFIG_PROFILE.
 	pr, hasProfileFlag := profileFlagValue(cmd)
 	if hasProfileFlag {
 		cfg.Profile = pr
+		// An explicit --profile must take precedence over authentication
+		// environment variables; see the matching comment in MustWorkspaceClient
+		// and issue #5096. We deliberately skip NormalizeDatabricksConfigFromEnv
+		// here: with --profile the host comes from the profile, not from
+		// DATABRICKS_HOST, so promoting that env var's ?o=/?a= query params
+		// would be wrong.
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+	} else {
+		auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
 	}
 
-	ctx := cmd.Context()
-	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
 	ctx = cmdctx.SetConfigUsed(ctx, cfg)
 	cmd.SetContext(ctx)
 
@@ -324,9 +332,21 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error {
 	profile, hasProfileFlag := profileFlagValue(cmd)
 	if hasProfileFlag {
 		cfg.Profile = profile
+		// An explicit --profile must take precedence over authentication
+		// environment variables (DATABRICKS_HOST, DATABRICKS_TOKEN, ...);
+		// see databrickscfg.ProfileAuthLoaders and issue #5096.
+		//
+		// We deliberately skip NormalizeDatabricksConfigFromEnv here: with
+		// --profile the host comes from the profile, not from DATABRICKS_HOST,
+		// so promoting that env var's ?o=/?a= query params would be wrong. The
+		// one edge this drops is a host-less profile combined with a SPOG-style
+		// DATABRICKS_HOST (https://host/?o=123): the workspace_id is no longer
+		// extracted from the query, which is an accepted trade-off for that
+		// unusual combination.
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+	} else {
+		auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
 	}
-
-	auth.NormalizeDatabricksConfigFromEnv(ctx, cfg)
 	resolveDefaultProfile(ctx, cfg)
 
 	_, isTargetFlagSet := targetFlagValue(cmd)

cmd/root/auth_test.go

@@ -442,6 +442,108 @@ token = flag-token
 	}
 }
 
+func TestMustWorkspaceClientProfileFlagOverridesAuthEnv(t *testing.T) {
+	testutil.CleanupEnvironment(t)
+
+	configFile := filepath.Join(t.TempDir(), ".databrickscfg")
+	err := os.WriteFile(configFile, []byte(`
+[tst-svc]
+host = https://tst.cloud.databricks.test
+token = tst-token
+`), 0o600)
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_CONFIG_FILE", configFile)
+	// direnv-style auth env vars pointing at a different (dev) workspace. Before
+	// the fix for #5096 these shadowed the profile selected with --profile.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	ctx := cmdio.MockDiscard(t.Context())
+	ctx = SkipLoadBundle(ctx)
+	cmd := New(ctx)
+
+	err = cmd.Flag("profile").Value.Set("tst-svc")
+	require.NoError(t, err)
+
+	err = MustWorkspaceClient(cmd, []string{})
+	require.NoError(t, err)
+
+	w := cmdctx.WorkspaceClient(cmd.Context())
+	require.NotNil(t, w)
+	// The explicitly selected profile must win over the auth env vars.
+	assert.Equal(t, "tst-svc", w.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", w.Config.Host)
+	assert.Equal(t, "tst-token", w.Config.Token)
+}
+
+func TestMustWorkspaceClientProfileFlagFillsAuthFromEnv(t *testing.T) {
+	testutil.CleanupEnvironment(t)
+
+	// A host-only profile relies on the environment for credentials. This is a
+	// common CI pattern: the host lives in .databrickscfg while DATABRICKS_TOKEN
+	// is injected by the runner. The profile must take precedence for the host,
+	// but the env must still fill the token the profile does not provide (#5096).
+	configFile := filepath.Join(t.TempDir(), ".databrickscfg")
+	err := os.WriteFile(configFile, []byte(`
+[host-only]
+host = https://tst.cloud.databricks.test
+`), 0o600)
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_CONFIG_FILE", configFile)
+	t.Setenv("DATABRICKS_TOKEN", "env-token")
+
+	ctx := cmdio.MockDiscard(t.Context())
+	ctx = SkipLoadBundle(ctx)
+	cmd := New(ctx)
+
+	err = cmd.Flag("profile").Value.Set("host-only")
+	require.NoError(t, err)
+
+	err = MustWorkspaceClient(cmd, []string{})
+	require.NoError(t, err)
+
+	w := cmdctx.WorkspaceClient(cmd.Context())
+	require.NotNil(t, w)
+	assert.Equal(t, "host-only", w.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", w.Config.Host)
+	// The token is not in the profile, so it is filled from the environment.
+	assert.Equal(t, "env-token", w.Config.Token)
+}
+
+func TestMustAccountClientProfileFlagOverridesAuthEnv(t *testing.T) {
+	testutil.CleanupEnvironment(t)
+
+	configFile := filepath.Join(t.TempDir(), ".databrickscfg")
+	err := os.WriteFile(configFile, []byte(`
+[acc-tst]
+host = https://accounts.cloud.databricks.test
+account_id = 1111
+token = tst-token
+`), 0o600)
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_CONFIG_FILE", configFile)
+	// Auth env vars pointing at a different account host. The profile must win.
+	t.Setenv("DATABRICKS_HOST", "https://accounts.dev.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	cmd := New(t.Context())
+	err = cmd.Flag("profile").Value.Set("acc-tst")
+	require.NoError(t, err)
+
+	err = MustAccountClient(cmd, []string{})
+	require.NoError(t, err)
+
+	a := cmdctx.AccountClient(cmd.Context())
+	require.NotNil(t, a)
+	// The explicitly selected profile must win over the auth env vars.
+	assert.Equal(t, "acc-tst", a.Config.Profile)
+	assert.Equal(t, "https://accounts.cloud.databricks.test", a.Config.Host)
+	assert.Equal(t, "tst-token", a.Config.Token)
+}
+
 func TestAccountClientOrPromptReturnsErrorForWrongHostType(t *testing.T) {
 	testutil.CleanupEnvironment(t)
 	t.Setenv("PATH", "")