From 31b7ad071c4b1f836a1170e369045f9ac4911f84 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 28 Feb 2025 14:12:15 +0100 Subject: [PATCH 01/90] Add the auth.EnvVars function --- libs/auth/env.go | 14 ++++++++++++++ libs/auth/env_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/libs/auth/env.go b/libs/auth/env.go index 5c0d2129297..f58f29ef7d8 100644 --- a/libs/auth/env.go +++ b/libs/auth/env.go @@ -38,3 +38,17 @@ func GetEnvFor(name string) (string, bool) { return "", false } + +func EnvVars() []string { + out := []string{} + + for _, attr := range config.ConfigAttributes { + if len(attr.EnvVars) == 0 { + continue + } + + out = append(out, attr.EnvVars[0]) + } + + return out +} diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go index 850110b6029..d7efd0ddc4f 100644 --- a/libs/auth/env_test.go +++ b/libs/auth/env_test.go @@ -79,3 +79,40 @@ func TestGetEnvFor(t *testing.T) { assert.False(t, ok) assert.Empty(t, out) } + +func TestAuthEnvVars(t *testing.T) { + expected := []string{ + "DATABRICKS_HOST", + "DATABRICKS_CLUSTER_ID", + "DATABRICKS_WAREHOUSE_ID", + "DATABRICKS_SERVERLESS_COMPUTE_ID", + "DATABRICKS_METADATA_SERVICE_URL", + "DATABRICKS_ACCOUNT_ID", + "DATABRICKS_TOKEN", + "DATABRICKS_USERNAME", + "DATABRICKS_PASSWORD", + "DATABRICKS_CONFIG_PROFILE", + "DATABRICKS_CONFIG_FILE", + "DATABRICKS_GOOGLE_SERVICE_ACCOUNT", + "GOOGLE_CREDENTIALS", + "DATABRICKS_AZURE_RESOURCE_ID", + "ARM_USE_MSI", + "ARM_CLIENT_SECRET", + "ARM_CLIENT_ID", + "ARM_TENANT_ID", + "ACTIONS_ID_TOKEN_REQUEST_URL", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN", + "ARM_ENVIRONMENT", + "DATABRICKS_AZURE_LOGIN_APP_ID", + "DATABRICKS_CLIENT_ID", + "DATABRICKS_CLIENT_SECRET", + "DATABRICKS_CLI_PATH", + "DATABRICKS_AUTH_TYPE", + "DATABRICKS_DEBUG_TRUNCATE_BYTES", + "DATABRICKS_DEBUG_HEADERS", + "DATABRICKS_RATE_LIMIT", + } + + out := EnvVars() + assert.Equal(t, expected, out) +} From bf234c4f5018bb3096aba920d8e281a5e53c5715 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 28 Feb 2025 14:41:37 +0100 Subject: [PATCH 02/90] Add the auth.ProcessEnv function --- internal/testutil/env.go | 31 ++++++++++++-------- libs/auth/env.go | 61 ++++++++++++++++++++++++++++++++++++++-- libs/auth/env_test.go | 36 +++++++++++++++++++++++- 3 files changed, 114 insertions(+), 14 deletions(-) diff --git a/internal/testutil/env.go b/internal/testutil/env.go index 598229655fd..0aa97d77991 100644 --- a/internal/testutil/env.go +++ b/internal/testutil/env.go @@ -13,19 +13,11 @@ import ( // The original environment is restored upon test completion. // Note: use of this function is incompatible with parallel execution. func CleanupEnvironment(t TestingT) { - // Restore environment when test finishes. - environ := os.Environ() - t.Cleanup(func() { - // Restore original environment. - for _, kv := range environ { - kvs := strings.SplitN(kv, "=", 2) - os.Setenv(kvs[0], kvs[1]) - } - }) - path := os.Getenv("PATH") pwd := os.Getenv("PWD") - os.Clearenv() + + // Clear all environment variables. + ClearEnvironment(t) // We use t.Setenv instead of os.Setenv because the former actively // prevents a test being run with t.Parallel. Modifying the environment @@ -38,6 +30,23 @@ func CleanupEnvironment(t TestingT) { } } +// ClearEnvironment sets up an empty environment with no environment variables set. +// The original environment is restored upon test completion. +// Note: use of this function is incompatible with parallel execution +func ClearEnvironment(t TestingT) { + // Restore environment when test finishes. + environ := os.Environ() + t.Cleanup(func() { + // Restore original environment. + for _, kv := range environ { + kvs := strings.SplitN(kv, "=", 2) + os.Setenv(kvs[0], kvs[1]) + } + }) + + os.Clearenv() +} + // Changes into specified directory for the duration of the test. // Returns the current working directory. func Chdir(t TestingT, dir string) string { diff --git a/libs/auth/env.go b/libs/auth/env.go index f58f29ef7d8..6b53d22a606 100644 --- a/libs/auth/env.go +++ b/libs/auth/env.go @@ -1,6 +1,14 @@ package auth -import "github.com/databricks/databricks-sdk-go/config" +import ( + "fmt" + "os" + "slices" + "sort" + "strings" + + "github.com/databricks/databricks-sdk-go/config" +) // Env generates the authentication environment variables we need to set for // downstream applications from the CLI to work correctly. @@ -39,7 +47,7 @@ func GetEnvFor(name string) (string, bool) { return "", false } -func EnvVars() []string { +func envVars() []string { out := []string{} for _, attr := range config.ConfigAttributes { @@ -52,3 +60,52 @@ func EnvVars() []string { return out } + +// ProcessEnv generates the environment variables can be set to authenticate downstream +// processes to use the same auth credentials as in cfg. +func ProcessEnv(cfg *config.Config) []string { + // We want child telemetry processes to inherit environment variables like $HOME or $HTTPS_PROXY + // because they influence auth resolution. + base := os.Environ() + + out := []string{} + authEnvVars := envVars() + + // Remove any existing auth environment variables. This is done because + // the CLI offers multiple modalities of configuring authentication like + // `--profile` or `DATABRICKS_CONFIG_PROFILE` or `profile: ` in the + // bundle config file. + // + // Each of these modalities have different priorities and thus we don't want + // any auth configuration to piggyback into the child process environment. + // + // This is a precaution to avoid conflicting auth configurations being passed + // to the child telemetry process. + // + // Normally this should be unnecessary because the SDK should error if multiple + // authentication methods have been configured. But there is no harm in doing this + // as a precaution. + for _, v := range base { + k, _, found := strings.Cut(v, "=") + if !found { + continue + } + if slices.Contains(authEnvVars, k) { + continue + } + out = append(out, v) + } + + // Now add the necessary authentication environment variables. + newEnv := Env(cfg) + for k, v := range newEnv { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + + // Sort the environment variables so that the output is deterministic. + sort.Slice(out, func(i, j int) bool { + return out[i] < out[j] + }) + + return out +} diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go index d7efd0ddc4f..cf771cce4dd 100644 --- a/libs/auth/env_test.go +++ b/libs/auth/env_test.go @@ -3,6 +3,7 @@ package auth import ( "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" ) @@ -113,6 +114,39 @@ func TestAuthEnvVars(t *testing.T) { "DATABRICKS_RATE_LIMIT", } - out := EnvVars() + out := envVars() + assert.Equal(t, expected, out) +} + +func TestAuthProcessEnv(t *testing.T) { + testutil.ClearEnvironment(t) + + // Environment variables that should be inherited by child processes. + t.Setenv("HOME", "/home/user") + t.Setenv("HTTPS_PROXY", "http://proxy.com") + + // Environment variables that should be cleaned up by process env: + t.Setenv("DATABRICKS_HOST", "https://test.com") + t.Setenv("DATABRICKS_TOKEN", "test-token") + t.Setenv("DATABRICKS_PASSWORD", "test-password") + t.Setenv("DATABRICKS_METADATA_SERVICE_URL", "http://somurl.com") + t.Setenv("ARM_USE_MSI", "true") + t.Setenv("ARM_TENANT_ID", "test-tenant-id") + t.Setenv("ARM_CLIENT_ID", "test-client-id") + t.Setenv("ARM_CLIENT_SECRET", "test-client-secret") + + in := &config.Config{ + Host: "https://newhost.com", + Token: "new-token", + } + + expected := []string{ + "DATABRICKS_HOST=https://newhost.com", + "DATABRICKS_TOKEN=new-token", + "HOME=/home/user", + "HTTPS_PROXY=http://proxy.com", + } + + out := ProcessEnv(in) assert.Equal(t, expected, out) } From 42f6ecf6d72ca8cc312ae7ec56adc681c69dab78 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 28 Feb 2025 14:47:16 +0100 Subject: [PATCH 03/90] - --- libs/auth/env_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go index cf771cce4dd..e04153ed308 100644 --- a/libs/auth/env_test.go +++ b/libs/auth/env_test.go @@ -125,7 +125,7 @@ func TestAuthProcessEnv(t *testing.T) { t.Setenv("HOME", "/home/user") t.Setenv("HTTPS_PROXY", "http://proxy.com") - // Environment variables that should be cleaned up by process env: + // Environment variables that should be cleaned up by ProcessEnv(): t.Setenv("DATABRICKS_HOST", "https://test.com") t.Setenv("DATABRICKS_TOKEN", "test-token") t.Setenv("DATABRICKS_PASSWORD", "test-password") From 4f43fb9acf8892cc8a95905e057e19ca42deb10d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 2 Mar 2025 18:11:46 +0100 Subject: [PATCH 04/90] [WIP] Add bundle exec --- acceptance/acceptance_test.go | 2 +- acceptance/bundle/exec/basic/output.txt | 5 ++ acceptance/bundle/exec/basic/script | 1 + acceptance/bundle/help/bundle/output.txt | 1 + cmd/bundle/bundle.go | 1 + cmd/bundle/exec.go | 109 +++++++++++++++++++++++ 6 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/exec/basic/output.txt create mode 100644 acceptance/bundle/exec/basic/script create mode 100644 cmd/bundle/exec.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 066a84299ec..f8b259643a3 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -41,7 +41,7 @@ var ( // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" // Then install your breakpoints and click "debug test" near TestAccept in VSCODE. // example: var SingleTest = "bundle/variables/empty" -var SingleTest = "" +var SingleTest = "bundle/exec/basic" // If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs // CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py). diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt new file mode 100644 index 00000000000..bc6165d261b --- /dev/null +++ b/acceptance/bundle/exec/basic/output.txt @@ -0,0 +1,5 @@ + +>>> errcode [CLI] bundle exec -- echo Hello, World! +Error: Please add a '--' separator. Usage: 'databricks bundle exec -- arg1 arg2 ...' + +Exit code: 1 diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script new file mode 100644 index 00000000000..f0d714b9a47 --- /dev/null +++ b/acceptance/bundle/exec/basic/script @@ -0,0 +1 @@ +trace errcode $CLI bundle exec echo "Hello,\ World" diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index fc6dd623dde..bb885c80e53 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -11,6 +11,7 @@ Available Commands: deploy Deploy bundle deployment Deployment related commands destroy Destroy deployed bundle resources + exec Execute a command using the same authentication context as the bundle generate Generate bundle configuration init Initialize using a bundle template open Open a resource in the browser diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index fb88cd7d05a..e0818c2f977 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -28,5 +28,6 @@ func New() *cobra.Command { cmd.AddCommand(newDebugCommand()) cmd.AddCommand(deployment.NewDeploymentCommand()) cmd.AddCommand(newOpenCommand()) + cmd.AddCommand(newExecCommand()) return cmd } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go new file mode 100644 index 00000000000..efc8164f59d --- /dev/null +++ b/cmd/bundle/exec.go @@ -0,0 +1,109 @@ +package bundle + +import ( + "bufio" + "fmt" + "os/exec" + "strings" + "sync" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/spf13/cobra" +) + +// TODO: Confirm that quoted strings are parsed as a single argument. +// TODO: test that -- works with flags as well. +// TODO CONTINUE: Making the bundle exec function work. +// TODO CONTINUE: Adding the scripts section to DABs. + +func newExecCommand() *cobra.Command { + variadicArgs := []string{} + + execCmd := &cobra.Command{ + Use: "exec", + Short: "Execute a command using the same authentication context as the bundle", + Args: cobra.MinimumNArgs(1), + Long: `Examples: + // TODO: Ensure that these multi work strings work with the exec command. +1. databricks bundle exec -- echo "Hello, world!" +2. databricks bundle exec -- /bin/bash -c "echo 'Hello, world!'" +3. databricks bundle exec -- uv run pytest"`, + RunE: func(cmd *cobra.Command, args []string) error { + if cmd.ArgsLenAtDash() != 0 { + return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) + } + + // Load the bundle configuration to get the authentication credentials + // set in the context. + // TODO: What happens when no bundle is configured? + _, diags := root.MustConfigureBundle(cmd) + if diags.HasError() { + return diags.Error() + } + + childCmd := exec.Command(args[1], args[2:]...) + childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + + // Create pipes for stdout and stderr + stdout, err := childCmd.StdoutPipe() + if err != nil { + return fmt.Errorf("Error creating stdout pipe: %w", err) + } + + stderr, err := childCmd.StderrPipe() + if err != nil { + return fmt.Errorf("Error creating stderr pipe: %w", err) + } + + // Start the command + if err := childCmd.Start(); err != nil { + return fmt.Errorf("Error starting command: %s\n", err) + } + + // Stream both stdout and stderr to the user. + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + }() + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + }() + + // Wait for the command to finish. + // TODO: Pretty exit codes? + // TODO: Make CLI return the same exit codes? + err = childCmd.Wait() + if exitErr, ok := err.(*exec.ExitError); ok { + return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) + } + + if err := childCmd.Wait(); err != nil { + return fmt.Errorf("Error waiting for command: %w", err) + } + + // Wait for the goroutines to finish printing to stdout and stderr. + wg.Wait() + + return nil + }, + } + + // TODO: Is this needed to make -- work with flags? + // execCmd.Flags().SetInterspersed(false) + + // TODO: func (c *Command) ArgsLenAtDash() int solves my problems here. + + return execCmd +} From 90c6ad0fdb8f84d96ca3853d0156b8f32ff6d541 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 10:52:31 +0100 Subject: [PATCH 05/90] - --- cmd/bundle/exec.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index efc8164f59d..11b38ef18e0 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -16,16 +16,15 @@ import ( // TODO: test that -- works with flags as well. // TODO CONTINUE: Making the bundle exec function work. // TODO CONTINUE: Adding the scripts section to DABs. +// TODO: Ensure that these multi word strings work with the exec command. Example: echo "Hello, world!" +// Or if it does not work, be sure why. Probably because string parsing is a part of the bash shell. func newExecCommand() *cobra.Command { - variadicArgs := []string{} - execCmd := &cobra.Command{ Use: "exec", Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), Long: `Examples: - // TODO: Ensure that these multi work strings work with the exec command. 1. databricks bundle exec -- echo "Hello, world!" 2. databricks bundle exec -- /bin/bash -c "echo 'Hello, world!'" 3. databricks bundle exec -- uv run pytest"`, From 6002e0d0402525e3c265fbb44f177ae91ef245bb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 14:01:41 +0100 Subject: [PATCH 06/90] some more tests --- acceptance/acceptance_test.go | 2 +- acceptance/bundle/exec/basic/databricks.yml | 2 ++ acceptance/bundle/exec/basic/output.txt | 5 +++-- acceptance/bundle/exec/basic/script | 2 +- acceptance/bundle/exec/no-separator/output.txt | 5 +++++ acceptance/bundle/exec/no-separator/script | 1 + cmd/bundle/exec.go | 8 ++++---- 7 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/exec/basic/databricks.yml create mode 100644 acceptance/bundle/exec/no-separator/output.txt create mode 100644 acceptance/bundle/exec/no-separator/script diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index f8b259643a3..066a84299ec 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -41,7 +41,7 @@ var ( // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" // Then install your breakpoints and click "debug test" near TestAccept in VSCODE. // example: var SingleTest = "bundle/variables/empty" -var SingleTest = "bundle/exec/basic" +var SingleTest = "" // If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs // CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py). diff --git a/acceptance/bundle/exec/basic/databricks.yml b/acceptance/bundle/exec/basic/databricks.yml new file mode 100644 index 00000000000..432311dab0d --- /dev/null +++ b/acceptance/bundle/exec/basic/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index bc6165d261b..d45ebfb8391 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -1,5 +1,6 @@ ->>> errcode [CLI] bundle exec -- echo Hello, World! -Error: Please add a '--' separator. Usage: 'databricks bundle exec -- arg1 arg2 ...' +>>> errcode [CLI] bundle exec -- echo hello +hello +Error: Error waiting for command: exec: Wait was already called Exit code: 1 diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index f0d714b9a47..c65cd19eab9 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1 +1 @@ -trace errcode $CLI bundle exec echo "Hello,\ World" +trace errcode $CLI bundle exec -- echo hello diff --git a/acceptance/bundle/exec/no-separator/output.txt b/acceptance/bundle/exec/no-separator/output.txt new file mode 100644 index 00000000000..ef7b6f1ce12 --- /dev/null +++ b/acceptance/bundle/exec/no-separator/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle exec echo hello +Error: Please add a '--' separator. Usage: 'databricks bundle exec -- echo hello' + +Exit code: 1 diff --git a/acceptance/bundle/exec/no-separator/script b/acceptance/bundle/exec/no-separator/script new file mode 100644 index 00000000000..050538fa4d0 --- /dev/null +++ b/acceptance/bundle/exec/no-separator/script @@ -0,0 +1 @@ +trace $CLI bundle exec echo hello diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 11b38ef18e0..daeb0c4a2b4 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -25,8 +25,8 @@ func newExecCommand() *cobra.Command { Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), Long: `Examples: -1. databricks bundle exec -- echo "Hello, world!" -2. databricks bundle exec -- /bin/bash -c "echo 'Hello, world!'" +1. databricks bundle exec -- echo hello +2. databricks bundle exec -- /bin/bash -c "echo hello"" 3. databricks bundle exec -- uv run pytest"`, RunE: func(cmd *cobra.Command, args []string) error { if cmd.ArgsLenAtDash() != 0 { @@ -41,7 +41,7 @@ func newExecCommand() *cobra.Command { return diags.Error() } - childCmd := exec.Command(args[1], args[2:]...) + childCmd := exec.Command(args[0], args[1:]...) childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) // Create pipes for stdout and stderr @@ -82,7 +82,7 @@ func newExecCommand() *cobra.Command { // Wait for the command to finish. // TODO: Pretty exit codes? - // TODO: Make CLI return the same exit codes? + // TODO: Make CLI return the same exit codes? It has to, that's a requirement. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) From 1b5ff4887315db3a2334a82646f8a9024b8085f9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 15:12:50 +0100 Subject: [PATCH 07/90] execute scripts from bundle root --- .vscode/settings.json | 5 ++++- acceptance/bundle/exec/basic/output.txt | 8 ++++---- acceptance/bundle/exec/basic/script | 4 +++- acceptance/bundle/exec/cwd/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/databricks.yml | 2 ++ acceptance/bundle/exec/cwd/output.txt | 8 ++++++++ acceptance/bundle/exec/cwd/script | 6 ++++++ acceptance/bundle/exec/no-auth/databricks.yml | 2 ++ acceptance/bundle/exec/no-auth/output.txt | 6 ++++++ acceptance/bundle/exec/no-auth/script | 8 ++++++++ acceptance/bundle/exec/no-bundle/output.txt | 5 +++++ acceptance/bundle/exec/no-bundle/script | 1 + cmd/bundle/exec.go | 19 ++++++++++++++----- 13 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 acceptance/bundle/exec/cwd/a/b/c/.gitkeep create mode 100644 acceptance/bundle/exec/cwd/databricks.yml create mode 100644 acceptance/bundle/exec/cwd/output.txt create mode 100644 acceptance/bundle/exec/cwd/script create mode 100644 acceptance/bundle/exec/no-auth/databricks.yml create mode 100644 acceptance/bundle/exec/no-auth/output.txt create mode 100644 acceptance/bundle/exec/no-auth/script create mode 100644 acceptance/bundle/exec/no-bundle/output.txt create mode 100644 acceptance/bundle/exec/no-bundle/script diff --git a/.vscode/settings.json b/.vscode/settings.json index f8b04f12692..f103538b7f3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,5 +17,8 @@ "python.envFile": "${workspaceRoot}/.env", "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------" + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "files.associations": { + "script": "shellscript" + } } diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index d45ebfb8391..c1be99c9e6d 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -1,6 +1,6 @@ ->>> errcode [CLI] bundle exec -- echo hello -hello -Error: Error waiting for command: exec: Wait was already called +>>> [CLI] bundle exec -- echo hello, world +hello, world -Exit code: 1 +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index c65cd19eab9..48d20c1bd9a 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1 +1,3 @@ -trace errcode $CLI bundle exec -- echo hello +trace $CLI bundle exec -- echo "hello, world" + +trace $CLI bundle exec -- pwd diff --git a/acceptance/bundle/exec/cwd/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/a/b/c/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/exec/cwd/databricks.yml b/acceptance/bundle/exec/cwd/databricks.yml new file mode 100644 index 00000000000..432311dab0d --- /dev/null +++ b/acceptance/bundle/exec/cwd/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt new file mode 100644 index 00000000000..e6496afab82 --- /dev/null +++ b/acceptance/bundle/exec/cwd/output.txt @@ -0,0 +1,8 @@ + +>>> cd a/b/c + +>>> pwd +[TMPDIR]/a/b/c + +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/script new file mode 100644 index 00000000000..73a0966d3fd --- /dev/null +++ b/acceptance/bundle/exec/cwd/script @@ -0,0 +1,6 @@ +trace cd a/b/c + +trace pwd + +# Scripts that bundle exec executes should run from the bundle root. +trace $CLI bundle exec -- pwd diff --git a/acceptance/bundle/exec/no-auth/databricks.yml b/acceptance/bundle/exec/no-auth/databricks.yml new file mode 100644 index 00000000000..432311dab0d --- /dev/null +++ b/acceptance/bundle/exec/no-auth/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt new file mode 100644 index 00000000000..fce8e3f613a --- /dev/null +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -- echo hello +hello + +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script new file mode 100644 index 00000000000..93715d44817 --- /dev/null +++ b/acceptance/bundle/exec/no-auth/script @@ -0,0 +1,8 @@ +export DATABRICKS_HOST="" +export DATABRICKS_TOKEN="" + +# Confirm that bundle exec works for commands that do not require authentication, +# even if authentication is not provided. +trace $CLI bundle exec -- echo hello + +trace $CLI bundle exec -- pwd diff --git a/acceptance/bundle/exec/no-bundle/output.txt b/acceptance/bundle/exec/no-bundle/output.txt new file mode 100644 index 00000000000..7030c5405ce --- /dev/null +++ b/acceptance/bundle/exec/no-bundle/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle exec -- echo hello +Error: unable to locate bundle root: databricks.yml not found + +Exit code: 1 diff --git a/acceptance/bundle/exec/no-bundle/script b/acceptance/bundle/exec/no-bundle/script new file mode 100644 index 00000000000..b872f65db1f --- /dev/null +++ b/acceptance/bundle/exec/no-bundle/script @@ -0,0 +1 @@ +trace $CLI bundle exec -- echo hello diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index daeb0c4a2b4..64bc70da57c 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -24,7 +24,13 @@ func newExecCommand() *cobra.Command { Use: "exec", Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), - Long: `Examples: + // TODO: format once we have all the documentation here. + // TODO: implement and pass the cwd environment variable. Maybe we can defer + // it until we have a scripts section. + Long: ` +Note: This command executes scripts + +Examples: 1. databricks bundle exec -- echo hello 2. databricks bundle exec -- /bin/bash -c "echo hello"" 3. databricks bundle exec -- uv run pytest"`, @@ -36,7 +42,7 @@ func newExecCommand() *cobra.Command { // Load the bundle configuration to get the authentication credentials // set in the context. // TODO: What happens when no bundle is configured? - _, diags := root.MustConfigureBundle(cmd) + b, diags := root.MustConfigureBundle(cmd) if diags.HasError() { return diags.Error() } @@ -44,7 +50,11 @@ func newExecCommand() *cobra.Command { childCmd := exec.Command(args[0], args[1:]...) childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // Create pipes for stdout and stderr + // Execute all scripts from the bundle root directory. + childCmd.Dir = b.BundleRootPath + + // Create pipes for stdout and stderr. + // TODO: Test streaming of this? Is there a way? stdout, err := childCmd.StdoutPipe() if err != nil { return fmt.Errorf("Error creating stdout pipe: %w", err) @@ -87,8 +97,7 @@ func newExecCommand() *cobra.Command { if exitErr, ok := err.(*exec.ExitError); ok { return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) } - - if err := childCmd.Wait(); err != nil { + if err != nil { return fmt.Errorf("Error waiting for command: %w", err) } From c442378f45eea4fb95fa0faecf585122b1af3645 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 15:19:18 +0100 Subject: [PATCH 08/90] clarify the cwd plan --- cmd/bundle/exec.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 64bc70da57c..bb6ecfc118d 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -12,12 +12,7 @@ import ( "github.com/spf13/cobra" ) -// TODO: Confirm that quoted strings are parsed as a single argument. // TODO: test that -- works with flags as well. -// TODO CONTINUE: Making the bundle exec function work. -// TODO CONTINUE: Adding the scripts section to DABs. -// TODO: Ensure that these multi word strings work with the exec command. Example: echo "Hello, world!" -// Or if it does not work, be sure why. Probably because string parsing is a part of the bash shell. func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ @@ -25,8 +20,6 @@ func newExecCommand() *cobra.Command { Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), // TODO: format once we have all the documentation here. - // TODO: implement and pass the cwd environment variable. Maybe we can defer - // it until we have a scripts section. Long: ` Note: This command executes scripts @@ -41,16 +34,24 @@ Examples: // Load the bundle configuration to get the authentication credentials // set in the context. - // TODO: What happens when no bundle is configured? b, diags := root.MustConfigureBundle(cmd) if diags.HasError() { return diags.Error() } childCmd := exec.Command(args[0], args[1:]...) + childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // Execute all scripts from the bundle root directory. + // Execute all scripts from the bundle root directory. This behavior can + // be surprising in isolation, but we do it to keep the behavior consistent + // for both cases: + // 1. One shot commands like `databricks bundle exec -- echo hello` + // 2. Scripts that are defined in the scripts section of the DAB. + // + // TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable + // that users can read to figure out the original CWD. I'll do that when + // adding support for the scripts section. childCmd.Dir = b.BundleRootPath // Create pipes for stdout and stderr. @@ -108,10 +109,8 @@ Examples: }, } - // TODO: Is this needed to make -- work with flags? + // TODO: Is this needed to make -- work with flags? What does this option do? // execCmd.Flags().SetInterspersed(false) - // TODO: func (c *Command) ArgsLenAtDash() int solves my problems here. - return execCmd } From 152d982c9bcd6c5254a7ab70e6a2c91f8caa5c65 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 16:58:57 +0100 Subject: [PATCH 09/90] add cases for the target flag --- .../exec/databricks-cli/basic/databricks.yml | 10 +++++ .../databricks-cli/basic/out.requests.txt | 41 +++++++++++++++++++ .../exec/databricks-cli/basic/output.txt | 24 +++++++++++ .../bundle/exec/databricks-cli/basic/script | 15 +++++++ .../bundle/exec/databricks-cli/test.toml | 2 + cmd/bundle/exec.go | 29 ++++++++++++- 6 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/exec/databricks-cli/basic/databricks.yml create mode 100644 acceptance/bundle/exec/databricks-cli/basic/out.requests.txt create mode 100644 acceptance/bundle/exec/databricks-cli/basic/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/basic/script create mode 100644 acceptance/bundle/exec/databricks-cli/test.toml diff --git a/acceptance/bundle/exec/databricks-cli/basic/databricks.yml b/acceptance/bundle/exec/databricks-cli/basic/databricks.yml new file mode 100644 index 00000000000..6b782c32579 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: foobar + +targets: + pat: + default: true + + oauth: + workspace: + client_id: client_id diff --git a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt new file mode 100644 index 00000000000..10a364e97a1 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt @@ -0,0 +1,41 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/oidc/.well-known/oauth-authorization-server" +} +{ + "headers": { + "Authorization": [ + "Basic Y2xpZW50X2lkOmNsaWVudC1zZWNyZXQ=" + ] + }, + "method": "POST", + "path": "/oidc/v1/token", + "raw_body": "grant_type=client_credentials\u0026scope=all-apis" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/basic/output.txt b/acceptance/bundle/exec/databricks-cli/basic/output.txt new file mode 100644 index 00000000000..48fcf8a6198 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle exec -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} + +>>> [CLI] bundle exec -t pat -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} + +>>> errcode [CLI] bundle exec -t pat -- databricks current-user me -t oauth +Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH +Error: Command exited with code: 1 + +Exit code: 1 + +>>> [CLI] bundle exec -t oauth -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/basic/script b/acceptance/bundle/exec/databricks-cli/basic/script new file mode 100644 index 00000000000..3a3c5efe988 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/script @@ -0,0 +1,15 @@ +# Default target +trace $CLI bundle exec -- databricks current-user me + +# Explicitly select default target +trace $CLI bundle exec -t pat -- databricks current-user me + +# Conflicting targets selected. This should fail because for the child command +# pat would be configured via environment variables and oauth via the CLI resulting +# in more than one authorization method configured. +trace errcode $CLI bundle exec -t pat -- databricks current-user me -t oauth + +# Explicitly select oauth target +export DATABRICKS_TOKEN="" +export DATABRICKS_CLIENT_SECRET="client-secret" +trace $CLI bundle exec -t oauth -- databricks current-user me diff --git a/acceptance/bundle/exec/databricks-cli/test.toml b/acceptance/bundle/exec/databricks-cli/test.toml new file mode 100644 index 00000000000..373138e0cdf --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/test.toml @@ -0,0 +1,2 @@ +RecordRequests = true +IncludeRequestHeaders = ["Authorization"] diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index bb6ecfc118d..4faac4039bf 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -14,6 +14,27 @@ import ( // TODO: test that -- works with flags as well. +// TODO: Can bundle auth be resolved twice? What about: +// databricks bundle exec -t foo -- databricks jobs list -t bar? +// OR +// databricks bundle exec -- databricks jobs list -t bar? +// OR +// databricks bundle exec -- databricks jobs list? +// OR +// databricks bundle exec -t foo -- databricks jobs list? +// +// For the first two, undefined behavior is fine. For the latter two we need to ensure +// that the target from exec is respected. +// +// Add tests for all four of these cases. +// --> Do I need similar tests for --profile as well? +// --> Also add test for what happens with a default target? + +// TODO: Add acceptance test that flags are indeed not parsed by the exec command and +// instead are parsed by the child command. + +// # TODO: Table test casing the target permutations + func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -41,7 +62,13 @@ Examples: childCmd := exec.Command(args[0], args[1:]...) - childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + // TODO: Test that this works correctly for all permutations. + // TODO: Do the same for profile flag. + // TODO: TODO: What happens here if a default target is resolved? When + // no targets are defined? + env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + childCmd.Env = env // Execute all scripts from the bundle root directory. This behavior can // be surprising in isolation, but we do it to keep the behavior consistent From f0d1dc56c8c448ef25b49c09023b06a52a185f23 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 17:40:04 +0100 Subject: [PATCH 10/90] fix replacement --- .../bundle/exec/databricks-cli/basic/out.requests.txt | 2 +- acceptance/bundle/exec/databricks-cli/basic/script | 2 +- .../exec/databricks-cli/profile-is-passed/databricks.yml | 2 ++ .../bundle/exec/databricks-cli/profile-is-passed/output.txt | 3 +++ .../bundle/exec/databricks-cli/profile-is-passed/script | 0 acceptance/bundle/exec/databricks-cli/test.toml | 6 ++++++ cmd/bundle/exec.go | 2 +- 7 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/script diff --git a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt index 10a364e97a1..26efbbb08d5 100644 --- a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt +++ b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt @@ -23,7 +23,7 @@ { "headers": { "Authorization": [ - "Basic Y2xpZW50X2lkOmNsaWVudC1zZWNyZXQ=" + "Basic [ENCODED_AUTH]" ] }, "method": "POST", diff --git a/acceptance/bundle/exec/databricks-cli/basic/script b/acceptance/bundle/exec/databricks-cli/basic/script index 3a3c5efe988..1f44239df46 100644 --- a/acceptance/bundle/exec/databricks-cli/basic/script +++ b/acceptance/bundle/exec/databricks-cli/basic/script @@ -11,5 +11,5 @@ trace errcode $CLI bundle exec -t pat -- databricks current-user me -t oauth # Explicitly select oauth target export DATABRICKS_TOKEN="" -export DATABRICKS_CLIENT_SECRET="client-secret" +export DATABRICKS_CLIENT_SECRET="client_secret" trace $CLI bundle exec -t oauth -- databricks current-user me diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml new file mode 100644 index 00000000000..432311dab0d --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt new file mode 100644 index 00000000000..a258c7073ac --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -0,0 +1,3 @@ +script: line 64: syntax error near unexpected token `)' + +Exit code: 2 diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/exec/databricks-cli/test.toml b/acceptance/bundle/exec/databricks-cli/test.toml index 373138e0cdf..24cf889ece1 100644 --- a/acceptance/bundle/exec/databricks-cli/test.toml +++ b/acceptance/bundle/exec/databricks-cli/test.toml @@ -1,2 +1,8 @@ RecordRequests = true IncludeRequestHeaders = ["Authorization"] + +# "client_id:client_secret" in base64 is Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=, expect to +# see this in Authorization header +[[Repls]] +Old = "Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=" +New = "[ENCODED_AUTH]" diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 4faac4039bf..5ec6a0d2804 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -33,7 +33,7 @@ import ( // TODO: Add acceptance test that flags are indeed not parsed by the exec command and // instead are parsed by the child command. -// # TODO: Table test casing the target permutations +// # TODO: Table test casing the profile flag permutations func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ From 080d1bf8b92ef3f80747284f04e0e6ca6daec5f8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:13:56 +0100 Subject: [PATCH 11/90] make the profile is passed test work --- .../profile-is-passed/.databrickscfg | 8 +++++ .../profile-is-passed/databricks.yml | 3 ++ .../profile-is-passed/out.requests.txt | 32 +++++++++++++++++++ .../profile-is-passed/output.txt | 13 ++++++-- .../databricks-cli/profile-is-passed/script | 14 ++++++++ cmd/bundle/exec.go | 8 +++++ 6 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg b/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg new file mode 100644 index 00000000000..fc39190cbed --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg @@ -0,0 +1,8 @@ +[someprofile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[myprofile] +host = $DATABRICKS_HOST +client_id = client_id +client_secret = client_secret diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml index 432311dab0d..5de7d1d9680 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml @@ -1,2 +1,5 @@ bundle: name: foobar + +workspace: + profile: someprofile diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt new file mode 100644 index 00000000000..2b214d4fd29 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt @@ -0,0 +1,32 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/oidc/.well-known/oauth-authorization-server" +} +{ + "headers": { + "Authorization": [ + "Basic [ENCODED_AUTH]" + ] + }, + "method": "POST", + "path": "/oidc/v1/token", + "raw_body": "grant_type=client_credentials\u0026scope=all-apis" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index a258c7073ac..d733dd0243b 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,3 +1,12 @@ -script: line 64: syntax error near unexpected token `)' -Exit code: 2 +>>> [CLI] bundle exec -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} + +>>> [CLI] bundle exec --profile myprofile -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script index e69de29bb2d..ad9280c5a6a 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script @@ -0,0 +1,14 @@ +# Replace placeholder with an actual host URL +envsubst < databricks.yml > out.yml && mv out.yml databricks.yml +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# This should use the profile configured in the bundle, i.e. PAT auth +trace $CLI bundle exec -- databricks current-user me + +# This should use myprofile, which uses oauth. +trace $CLI bundle exec --profile myprofile -- databricks current-user me diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 5ec6a0d2804..b5a52f6090b 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -68,6 +68,14 @@ Examples: // TODO: TODO: What happens here if a default target is resolved? When // no targets are defined? env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + + // If the bundle has a profile, explicitly pass it to the child command. + // This is unnecessary for tools that follow the unified authentication spec. + // However, because the CLI can read the profile from the bundle itself, we + // need to pass it explicitly. + if b.Config.Workspace.Profile != "" { + env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) + } childCmd.Env = env // Execute all scripts from the bundle root directory. This behavior can From 58d28a9c92c652ada851f4d5680f4d4ab8807bcc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:26:17 +0100 Subject: [PATCH 12/90] add more tests for profile and target --- .../{basic => target-is-passed}/databricks.yml | 0 .../{basic => target-is-passed}/out.requests.txt | 0 .../{basic => target-is-passed}/output.txt | 0 .../{basic => target-is-passed}/script | 0 bundle/config/mutator/default_target.go | 4 +++- cmd/bundle/exec.go | 16 +++++++++++----- 6 files changed, 14 insertions(+), 6 deletions(-) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/databricks.yml (100%) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/out.requests.txt (100%) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/output.txt (100%) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/script (100%) diff --git a/acceptance/bundle/exec/databricks-cli/basic/databricks.yml b/acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/databricks.yml rename to acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/out.requests.txt rename to acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/basic/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/output.txt rename to acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/basic/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/script rename to acceptance/bundle/exec/databricks-cli/target-is-passed/script diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index 73d99002a06..781c075ecdb 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -13,11 +13,13 @@ type defineDefaultTarget struct { name string } +const DefaultTargetName = "default" + // DefineDefaultTarget adds a target named "default" // to the configuration if none have been defined. func DefineDefaultTarget() bundle.Mutator { return &defineDefaultTarget{ - name: "default", + name: DefaultTargetName, } } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index b5a52f6090b..acf914f598d 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" "github.com/spf13/cobra" @@ -63,19 +64,24 @@ Examples: childCmd := exec.Command(args[0], args[1:]...) env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // TODO: Test that this works correctly for all permutations. - // TODO: Do the same for profile flag. - // TODO: TODO: What happens here if a default target is resolved? When - // no targets are defined? - env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + + // If user has specified a target, pass it to the child command. If the + // target is the default target, we don't need to pass it explicitly since + // the CLI will use the default target by default. + // This is only useful for when the Databricks CLI is the child command. + if b.Config.Bundle.Target != mutator.DefaultTargetName { + env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + } // If the bundle has a profile, explicitly pass it to the child command. // This is unnecessary for tools that follow the unified authentication spec. // However, because the CLI can read the profile from the bundle itself, we // need to pass it explicitly. + // This is only useful for when the Databricks CLI is the child command. if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } + childCmd.Env = env // Execute all scripts from the bundle root directory. This behavior can From 993956294aa35e707556bf486706f33ca9a85f51 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:35:03 +0100 Subject: [PATCH 13/90] added flags are not parsed check --- acceptance/bundle/exec/basic/output.txt | 3 +++ acceptance/bundle/exec/basic/script | 3 +++ cmd/bundle/exec.go | 26 ------------------------- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index c1be99c9e6d..01d3479f01f 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -4,3 +4,6 @@ hello, world >>> [CLI] bundle exec -- pwd [TMPDIR] + +>>> [CLI] bundle exec -- echo --help +--help diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 48d20c1bd9a..fb07fbfb6ff 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1,3 +1,6 @@ trace $CLI bundle exec -- echo "hello, world" trace $CLI bundle exec -- pwd + +# The CLI should not parse the --help flag and should pass it as is to echo. +trace $CLI bundle exec -- echo --help diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index acf914f598d..39224955af1 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -13,29 +13,6 @@ import ( "github.com/spf13/cobra" ) -// TODO: test that -- works with flags as well. - -// TODO: Can bundle auth be resolved twice? What about: -// databricks bundle exec -t foo -- databricks jobs list -t bar? -// OR -// databricks bundle exec -- databricks jobs list -t bar? -// OR -// databricks bundle exec -- databricks jobs list? -// OR -// databricks bundle exec -t foo -- databricks jobs list? -// -// For the first two, undefined behavior is fine. For the latter two we need to ensure -// that the target from exec is respected. -// -// Add tests for all four of these cases. -// --> Do I need similar tests for --profile as well? -// --> Also add test for what happens with a default target? - -// TODO: Add acceptance test that flags are indeed not parsed by the exec command and -// instead are parsed by the child command. - -// # TODO: Table test casing the profile flag permutations - func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -150,8 +127,5 @@ Examples: }, } - // TODO: Is this needed to make -- work with flags? What does this option do? - // execCmd.Flags().SetInterspersed(false) - return execCmd } From 05cd18c0be84c92c3f34de681fa4f17a22a835f6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:52:02 +0100 Subject: [PATCH 14/90] exit code done --- acceptance/bundle/exec/basic/output.txt | 5 +++++ acceptance/bundle/exec/basic/script | 3 +++ .../target-is-passed/output.txt | 2 +- cmd/bundle/exec.go | 20 ++++++++++++++++--- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 01d3479f01f..3584582dba2 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -7,3 +7,8 @@ hello, world >>> [CLI] bundle exec -- echo --help --help + +>>> [CLI] bundle exec -- bash -c exit 5 +Error: Running "bash -c exit 5" failed with exit code: 5 + +Exit code: 1 diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index fb07fbfb6ff..41c57725f40 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -4,3 +4,6 @@ trace $CLI bundle exec -- pwd # The CLI should not parse the --help flag and should pass it as is to echo. trace $CLI bundle exec -- echo --help + +# The error message should include the exit code. +trace $CLI bundle exec -- bash -c "exit 5" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 48fcf8a6198..a17f177be62 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -13,7 +13,7 @@ >>> errcode [CLI] bundle exec -t pat -- databricks current-user me -t oauth Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: Command exited with code: 1 +Error: Running "databricks current-user me -t oauth" failed with exit code: 1 Exit code: 1 diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 39224955af1..bd96b41deef 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -13,6 +13,15 @@ import ( "github.com/spf13/cobra" ) +type exitCodeErr struct { + exitCode int + args []string +} + +func (e *exitCodeErr) Error() string { + return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) +} + func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -110,11 +119,16 @@ Examples: }() // Wait for the command to finish. - // TODO: Pretty exit codes? - // TODO: Make CLI return the same exit codes? It has to, that's a requirement. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { - return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) + // We don't propagate the exit code as is because exit codes for + // the CLI have not been standardized yet. At some point in the + // future we might want to associate specific exit codes with + // specific classes of errors. + return &exitCodeErr{ + exitCode: exitErr.ExitCode(), + args: args, + } } if err != nil { return fmt.Errorf("Error waiting for command: %w", err) From a2748d531b31de50f7e46d729937d380c3c5c4af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:19:39 +0100 Subject: [PATCH 15/90] cleanup --- acceptance/bundle/exec/basic/output.txt | 3 ++ acceptance/bundle/exec/basic/script | 5 +++- cmd/bundle/exec.go | 37 +++++++++++++++++++------ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 3584582dba2..9283131d6f1 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -12,3 +12,6 @@ hello, world Error: Running "bash -c exit 5" failed with exit code: 5 Exit code: 1 + +>>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr +hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 41c57725f40..ebe76884bd0 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -6,4 +6,7 @@ trace $CLI bundle exec -- pwd trace $CLI bundle exec -- echo --help # The error message should include the exit code. -trace $CLI bundle exec -- bash -c "exit 5" +errcode trace $CLI bundle exec -- bash -c "exit 5" + +# stderr should also be shown in the output. +trace $CLI bundle exec -- bash -c "echo hello > /dev/stderr" diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index bd96b41deef..fbb4d25476f 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -27,11 +27,12 @@ func newExecCommand() *cobra.Command { Use: "exec", Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), - // TODO: format once we have all the documentation here. - Long: ` -Note: This command executes scripts + Long: `Execute a command using the same authentication context as the bundle -Examples: +The current working directory of the provided command will be set to the root +of the bundle. + +Example usage: 1. databricks bundle exec -- echo hello 2. databricks bundle exec -- /bin/bash -c "echo hello"" 3. databricks bundle exec -- uv run pytest"`, @@ -81,8 +82,8 @@ Examples: // adding support for the scripts section. childCmd.Dir = b.BundleRootPath - // Create pipes for stdout and stderr. - // TODO: Test streaming of this? Is there a way? + // Create pipes to stream the stdout and stderr output from the child + // process. stdout, err := childCmd.StdoutPipe() if err != nil { return fmt.Errorf("Error creating stdout pipe: %w", err) @@ -98,15 +99,21 @@ Examples: return fmt.Errorf("Error starting command: %s\n", err) } - // Stream both stdout and stderr to the user. + // Stream both stdout and stderr to the user. We do this so that the user + // does not have to wait for the command to finish before seeing the output. var wg sync.WaitGroup + var stdoutErr, stderrErr error wg.Add(2) go func() { defer wg.Done() scanner := bufio.NewScanner(stdout) for scanner.Scan() { - fmt.Println(scanner.Text()) + _, err = cmd.OutOrStdout().Write([]byte(scanner.Text() + "\n")) + if err != nil { + stdoutErr = fmt.Errorf("Error writing to stdout: %w", err) + return + } } }() @@ -114,10 +121,22 @@ Examples: defer wg.Done() scanner := bufio.NewScanner(stderr) for scanner.Scan() { - fmt.Println(scanner.Text()) + _, err := cmd.ErrOrStderr().Write([]byte(scanner.Text() + "\n")) + if err != nil { + stderrErr = fmt.Errorf("Error writing to stderr: %w", err) + return + } } }() + if stdoutErr != nil { + return stdoutErr + } + + if stderrErr != nil { + return stderrErr + } + // Wait for the command to finish. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { From 1996d3f824ef52c6a9f0715fd171e4285ee4ab81 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:29:19 +0100 Subject: [PATCH 16/90] do not run on cloud --- acceptance/bundle/exec/no-auth/script | 4 ++-- acceptance/bundle/exec/test.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/exec/test.toml diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index 93715d44817..0378ab26630 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -1,5 +1,5 @@ -export DATABRICKS_HOST="" -export DATABRICKS_TOKEN="" +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN # Confirm that bundle exec works for commands that do not require authentication, # even if authentication is not provided. diff --git a/acceptance/bundle/exec/test.toml b/acceptance/bundle/exec/test.toml new file mode 100644 index 00000000000..4564a92fedc --- /dev/null +++ b/acceptance/bundle/exec/test.toml @@ -0,0 +1,2 @@ +Cloud = false +Local = true From 88b6dc8e16a366dc2da91e3368f88ef9ca3d7da1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:31:41 +0100 Subject: [PATCH 17/90] use $CLI instead of databricks --- .../exec/databricks-cli/profile-is-passed/output.txt | 4 ++-- .../exec/databricks-cli/profile-is-passed/script | 4 ++-- .../exec/databricks-cli/target-is-passed/output.txt | 10 +++++----- .../bundle/exec/databricks-cli/target-is-passed/script | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index d733dd0243b..06cdd1c4400 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,11 +1,11 @@ ->>> [CLI] bundle exec -- databricks current-user me +>>> [CLI] bundle exec -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" } ->>> [CLI] bundle exec --profile myprofile -- databricks current-user me +>>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script index ad9280c5a6a..33e07494c61 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script @@ -8,7 +8,7 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # This should use the profile configured in the bundle, i.e. PAT auth -trace $CLI bundle exec -- databricks current-user me +trace $CLI bundle exec -- $CLI current-user me # This should use myprofile, which uses oauth. -trace $CLI bundle exec --profile myprofile -- databricks current-user me +trace $CLI bundle exec --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index a17f177be62..84a9653e928 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -1,23 +1,23 @@ ->>> [CLI] bundle exec -- databricks current-user me +>>> [CLI] bundle exec -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" } ->>> [CLI] bundle exec -t pat -- databricks current-user me +>>> [CLI] bundle exec -t pat -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" } ->>> errcode [CLI] bundle exec -t pat -- databricks current-user me -t oauth +>>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: Running "databricks current-user me -t oauth" failed with exit code: 1 +Error: Running "[CLI] current-user me -t oauth" failed with exit code: 1 Exit code: 1 ->>> [CLI] bundle exec -t oauth -- databricks current-user me +>>> [CLI] bundle exec -t oauth -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script index 1f44239df46..9d2eb2fe7ab 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/script @@ -1,15 +1,15 @@ # Default target -trace $CLI bundle exec -- databricks current-user me +trace $CLI bundle exec -- $CLI current-user me # Explicitly select default target -trace $CLI bundle exec -t pat -- databricks current-user me +trace $CLI bundle exec -t pat -- $CLI current-user me # Conflicting targets selected. This should fail because for the child command # pat would be configured via environment variables and oauth via the CLI resulting # in more than one authorization method configured. -trace errcode $CLI bundle exec -t pat -- databricks current-user me -t oauth +trace errcode $CLI bundle exec -t pat -- $CLI current-user me -t oauth # Explicitly select oauth target export DATABRICKS_TOKEN="" export DATABRICKS_CLIENT_SECRET="client_secret" -trace $CLI bundle exec -t oauth -- databricks current-user me +trace $CLI bundle exec -t oauth -- $CLI current-user me From fd2600cbea9438dded4503062c95026ccbb34cea Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:32:43 +0100 Subject: [PATCH 18/90] return stdout / stderr errors after --- cmd/bundle/exec.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index fbb4d25476f..f74458d06a4 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -129,14 +129,6 @@ Example usage: } }() - if stdoutErr != nil { - return stdoutErr - } - - if stderrErr != nil { - return stderrErr - } - // Wait for the command to finish. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { @@ -156,6 +148,14 @@ Example usage: // Wait for the goroutines to finish printing to stdout and stderr. wg.Wait() + if stdoutErr != nil { + return stdoutErr + } + + if stderrErr != nil { + return stderrErr + } + return nil }, } From 93c5e3f2ae648afd701cb958d80106ce9ea7ef0f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:50:56 +0100 Subject: [PATCH 19/90] simplify streaming --- cmd/bundle/exec.go | 59 ++++------------------------------------------ 1 file changed, 4 insertions(+), 55 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index f74458d06a4..794d2aa0ece 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -1,11 +1,9 @@ package bundle import ( - "bufio" "fmt" "os/exec" "strings" - "sync" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" @@ -82,55 +80,17 @@ Example usage: // adding support for the scripts section. childCmd.Dir = b.BundleRootPath - // Create pipes to stream the stdout and stderr output from the child - // process. - stdout, err := childCmd.StdoutPipe() - if err != nil { - return fmt.Errorf("Error creating stdout pipe: %w", err) - } - - stderr, err := childCmd.StderrPipe() - if err != nil { - return fmt.Errorf("Error creating stderr pipe: %w", err) - } + // Stream the stdout and stderr of the child process directly. + childCmd.Stdout = cmd.OutOrStdout() + childCmd.Stderr = cmd.ErrOrStderr() // Start the command if err := childCmd.Start(); err != nil { return fmt.Errorf("Error starting command: %s\n", err) } - // Stream both stdout and stderr to the user. We do this so that the user - // does not have to wait for the command to finish before seeing the output. - var wg sync.WaitGroup - var stdoutErr, stderrErr error - wg.Add(2) - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - _, err = cmd.OutOrStdout().Write([]byte(scanner.Text() + "\n")) - if err != nil { - stdoutErr = fmt.Errorf("Error writing to stdout: %w", err) - return - } - } - }() - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - _, err := cmd.ErrOrStderr().Write([]byte(scanner.Text() + "\n")) - if err != nil { - stderrErr = fmt.Errorf("Error writing to stderr: %w", err) - return - } - } - }() - // Wait for the command to finish. - err = childCmd.Wait() + err := childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { // We don't propagate the exit code as is because exit codes for // the CLI have not been standardized yet. At some point in the @@ -145,17 +105,6 @@ Example usage: return fmt.Errorf("Error waiting for command: %w", err) } - // Wait for the goroutines to finish printing to stdout and stderr. - wg.Wait() - - if stdoutErr != nil { - return stdoutErr - } - - if stderrErr != nil { - return stderrErr - } - return nil }, } From bbbbf30db3b41dceddc847af7b75e99161d1e8bc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:51:39 +0100 Subject: [PATCH 20/90] - --- .vscode/settings.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index f103538b7f3..f8b04f12692 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,8 +17,5 @@ "python.envFile": "${workspaceRoot}/.env", "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "files.associations": { - "script": "shellscript" - } + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------" } From eb58d11f9b247a248e11f3da59595615eeb4fa15 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:52:45 +0100 Subject: [PATCH 21/90] - --- acceptance/bundle/exec/basic/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index ebe76884bd0..e0e238c43d2 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -8,5 +8,5 @@ trace $CLI bundle exec -- echo --help # The error message should include the exit code. errcode trace $CLI bundle exec -- bash -c "exit 5" -# stderr should also be shown in the output. +# stderr should also be passed through. trace $CLI bundle exec -- bash -c "echo hello > /dev/stderr" From 007a714750ee2347728a85ff86d45c49b6fdf835 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:56:29 +0100 Subject: [PATCH 22/90] - --- acceptance/bundle/exec/databricks-cli/target-is-passed/script | 2 +- acceptance/bundle/exec/no-auth/script | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script index 9d2eb2fe7ab..238c36ae4b8 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/script @@ -10,6 +10,6 @@ trace $CLI bundle exec -t pat -- $CLI current-user me trace errcode $CLI bundle exec -t pat -- $CLI current-user me -t oauth # Explicitly select oauth target -export DATABRICKS_TOKEN="" +unset DATABRICKS_TOKEN export DATABRICKS_CLIENT_SECRET="client_secret" trace $CLI bundle exec -t oauth -- $CLI current-user me diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index 0378ab26630..a1758ce997a 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -4,5 +4,4 @@ unset DATABRICKS_TOKEN # Confirm that bundle exec works for commands that do not require authentication, # even if authentication is not provided. trace $CLI bundle exec -- echo hello - trace $CLI bundle exec -- pwd From 0c9fbf7b236510d208183b8c358b9e33fa7a7797 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:58:14 +0100 Subject: [PATCH 23/90] - --- bundle/config/mutator/default_target.go | 4 ++-- cmd/bundle/exec.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index 781c075ecdb..c2e9e8acf27 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -13,13 +13,13 @@ type defineDefaultTarget struct { name string } -const DefaultTargetName = "default" +const DefaultTargetPlaceholder = "default" // DefineDefaultTarget adds a target named "default" // to the configuration if none have been defined. func DefineDefaultTarget() bundle.Mutator { return &defineDefaultTarget{ - name: DefaultTargetName, + name: DefaultTargetPlaceholder, } } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 794d2aa0ece..4c62ec292fd 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -54,7 +54,7 @@ Example usage: // target is the default target, we don't need to pass it explicitly since // the CLI will use the default target by default. // This is only useful for when the Databricks CLI is the child command. - if b.Config.Bundle.Target != mutator.DefaultTargetName { + if b.Config.Bundle.Target != mutator.DefaultTargetPlaceholder { env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) } From edf705188d936f48113de822f2662b7b360f4c23 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:06:24 +0100 Subject: [PATCH 24/90] more cleanup --- cmd/bundle/exec.go | 33 ++++++++++++++++++--------------- internal/testutil/env.go | 31 +++++++++++-------------------- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 4c62ec292fd..1463aa20278 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -31,9 +31,9 @@ The current working directory of the provided command will be set to the root of the bundle. Example usage: -1. databricks bundle exec -- echo hello -2. databricks bundle exec -- /bin/bash -c "echo hello"" -3. databricks bundle exec -- uv run pytest"`, +1. databricks bundle exec -- echo "hello, world" +2. databricks bundle exec -- /bin/bash -c "echo hello" +3. databricks bundle exec -- uv run pytest`, RunE: func(cmd *cobra.Command, args []string) error { if cmd.ArgsLenAtDash() != 0 { return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) @@ -50,19 +50,20 @@ Example usage: env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // If user has specified a target, pass it to the child command. If the - // target is the default target, we don't need to pass it explicitly since - // the CLI will use the default target by default. + // If user has specified a target, pass it to the child command. DABs + // defines a "default" target which is a placeholder if no target is defined. + // If that's the case, i.e. no targets are defined, then do not pass the target. + // // This is only useful for when the Databricks CLI is the child command. if b.Config.Bundle.Target != mutator.DefaultTargetPlaceholder { env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) } - // If the bundle has a profile, explicitly pass it to the child command. - // This is unnecessary for tools that follow the unified authentication spec. - // However, because the CLI can read the profile from the bundle itself, we - // need to pass it explicitly. - // This is only useful for when the Databricks CLI is the child command. + // If the bundle has a profile configured, explicitly pass it to the child command. + // + // This is only useful for when the Databricks CLI is the child command, + // since if we do not explicitly pass the profile, the CLI will use the + // profile configured in the bundle YAML configuration (if any).We don't propagate the exit code as is because exit codes if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } @@ -92,10 +93,12 @@ Example usage: // Wait for the command to finish. err := childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { - // We don't propagate the exit code as is because exit codes for - // the CLI have not been standardized yet. At some point in the - // future we might want to associate specific exit codes with - // specific classes of errors. + // We don't make the parent CLI process exit with the same exit code + // as the child process because the exit codes for the CLI have not + // been standardized yet. + // + // This keeps the door open for us to associate specific exit codes + // with specific classes of errors in the future. return &exitCodeErr{ exitCode: exitErr.ExitCode(), args: args, diff --git a/internal/testutil/env.go b/internal/testutil/env.go index 0aa97d77991..598229655fd 100644 --- a/internal/testutil/env.go +++ b/internal/testutil/env.go @@ -13,11 +13,19 @@ import ( // The original environment is restored upon test completion. // Note: use of this function is incompatible with parallel execution. func CleanupEnvironment(t TestingT) { + // Restore environment when test finishes. + environ := os.Environ() + t.Cleanup(func() { + // Restore original environment. + for _, kv := range environ { + kvs := strings.SplitN(kv, "=", 2) + os.Setenv(kvs[0], kvs[1]) + } + }) + path := os.Getenv("PATH") pwd := os.Getenv("PWD") - - // Clear all environment variables. - ClearEnvironment(t) + os.Clearenv() // We use t.Setenv instead of os.Setenv because the former actively // prevents a test being run with t.Parallel. Modifying the environment @@ -30,23 +38,6 @@ func CleanupEnvironment(t TestingT) { } } -// ClearEnvironment sets up an empty environment with no environment variables set. -// The original environment is restored upon test completion. -// Note: use of this function is incompatible with parallel execution -func ClearEnvironment(t TestingT) { - // Restore environment when test finishes. - environ := os.Environ() - t.Cleanup(func() { - // Restore original environment. - for _, kv := range environ { - kvs := strings.SplitN(kv, "=", 2) - os.Setenv(kvs[0], kvs[1]) - } - }) - - os.Clearenv() -} - // Changes into specified directory for the duration of the test. // Returns the current working directory. func Chdir(t TestingT, dir string) string { From 769acf7fd3ffb1730850a45c0f0db276d2f8a4af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:07:51 +0100 Subject: [PATCH 25/90] - --- cmd/bundle/exec.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 1463aa20278..8703b4fbc7a 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -39,8 +39,7 @@ Example usage: return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) } - // Load the bundle configuration to get the authentication credentials - // set in the context. + // Load the bundle configuration to get the authentication credentials. b, diags := root.MustConfigureBundle(cmd) if diags.HasError() { return diags.Error() From 406103623d56c867c1aa25d13277d96cb05f482f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:08:33 +0100 Subject: [PATCH 26/90] - --- cmd/bundle/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 8703b4fbc7a..0010d62312d 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -50,7 +50,7 @@ Example usage: env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. DABs - // defines a "default" target which is a placeholder if no target is defined. + // defines a "default" target which is a placeholder for when no target is defined. // If that's the case, i.e. no targets are defined, then do not pass the target. // // This is only useful for when the Databricks CLI is the child command. From c0d34f7827d42faf4b488793c0d28e76849233e2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:09:20 +0100 Subject: [PATCH 27/90] - --- cmd/bundle/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 0010d62312d..3fd2b37d313 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -62,7 +62,7 @@ Example usage: // // This is only useful for when the Databricks CLI is the child command, // since if we do not explicitly pass the profile, the CLI will use the - // profile configured in the bundle YAML configuration (if any).We don't propagate the exit code as is because exit codes + // profile configured in the bundle YAML configuration (if any). if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } From 112a9de90a5dbe97bea17fc09e20e0a46141b403 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:10:27 +0100 Subject: [PATCH 28/90] - --- cmd/bundle/exec.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 3fd2b37d313..7dd831a2004 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -71,9 +71,9 @@ Example usage: // Execute all scripts from the bundle root directory. This behavior can // be surprising in isolation, but we do it to keep the behavior consistent - // for both cases: + // for both these cases: // 1. One shot commands like `databricks bundle exec -- echo hello` - // 2. Scripts that are defined in the scripts section of the DAB. + // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. // // TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable // that users can read to figure out the original CWD. I'll do that when From 647dab0a6660d619824836b58d0eeb857669abcd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:16:12 +0100 Subject: [PATCH 29/90] some cleanup --- .../bundle/exec/cmd-not-found/databricks.yml | 2 ++ .../bundle/exec/cmd-not-found/output.txt | 5 ++++ acceptance/bundle/exec/cmd-not-found/script | 1 + cmd/bundle/exec.go | 23 ++++++++++++------- 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/exec/cmd-not-found/databricks.yml create mode 100644 acceptance/bundle/exec/cmd-not-found/output.txt create mode 100644 acceptance/bundle/exec/cmd-not-found/script diff --git a/acceptance/bundle/exec/cmd-not-found/databricks.yml b/acceptance/bundle/exec/cmd-not-found/databricks.yml new file mode 100644 index 00000000000..432311dab0d --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt new file mode 100644 index 00000000000..36db6466769 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 +Error: Running "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH + +Exit code: 1 diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/exec/cmd-not-found/script new file mode 100644 index 00000000000..4a5d442f3a4 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/script @@ -0,0 +1 @@ +trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 7dd831a2004..dbd7d115692 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -20,6 +20,15 @@ func (e *exitCodeErr) Error() string { return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) } +type runErr struct { + err error + args []string +} + +func (e *runErr) Error() string { + return fmt.Sprintf("Running %q failed: %s", strings.Join(e.args, " "), e.err) +} + func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -84,13 +93,8 @@ Example usage: childCmd.Stdout = cmd.OutOrStdout() childCmd.Stderr = cmd.ErrOrStderr() - // Start the command - if err := childCmd.Start(); err != nil { - return fmt.Errorf("Error starting command: %s\n", err) - } - - // Wait for the command to finish. - err := childCmd.Wait() + // Run the command. + err := childCmd.Run() if exitErr, ok := err.(*exec.ExitError); ok { // We don't make the parent CLI process exit with the same exit code // as the child process because the exit codes for the CLI have not @@ -104,7 +108,10 @@ Example usage: } } if err != nil { - return fmt.Errorf("Error waiting for command: %w", err) + return &runErr{ + err: err, + args: args, + } } return nil From be3be9cb2f1a212c5c881c6018dd6fbd0416b8d4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Mar 2025 18:22:12 +0100 Subject: [PATCH 30/90] try streaming output --- .../bundle/exec/cmd-not-found/output.txt | 2 +- .../profile-is-passed/output.txt | 8 +- .../target-is-passed/output.txt | 12 +-- cmd/bundle/exec.go | 76 ++++++++++++++++--- 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index 36db6466769..8344948b44d 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ >>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: Running "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH +Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH Exit code: 1 diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index 06cdd1c4400..1c1b7405822 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,12 +1,12 @@ >>> [CLI] bundle exec -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } >>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 84a9653e928..0fce7c1a728 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -1,14 +1,14 @@ >>> [CLI] bundle exec -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } >>> [CLI] bundle exec -t pat -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } >>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth @@ -19,6 +19,6 @@ Exit code: 1 >>> [CLI] bundle exec -t oauth -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index dbd7d115692..e914f88448f 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -1,9 +1,11 @@ package bundle import ( + "bufio" "fmt" "os/exec" "strings" + "sync" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" @@ -89,12 +91,71 @@ Example usage: // adding support for the scripts section. childCmd.Dir = b.BundleRootPath - // Stream the stdout and stderr of the child process directly. - childCmd.Stdout = cmd.OutOrStdout() - childCmd.Stderr = cmd.ErrOrStderr() + stdout, err := childCmd.StdoutPipe() + if err != nil { + return fmt.Errorf("creating stdout pipe failed: %w", err) + } + + stderr, err := childCmd.StderrPipe() + if err != nil { + return fmt.Errorf("creating stderr pipe failed: %w", err) + } + + // Start the child command. + err = childCmd.Start() + if err != nil { + return fmt.Errorf("starting %q failed: %w", strings.Join(args, " "), err) + } + + var wg sync.WaitGroup + wg.Add(2) + + var stdoutErr error + go func() { + reader := bufio.NewReader(stdout) + line, err := reader.ReadString('\n') + for err == nil { + _, err = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", strings.TrimSpace(line)) + if err != nil { + stdoutErr = err + break + } + line, err = reader.ReadString('\n') + } - // Run the command. - err := childCmd.Run() + wg.Done() + }() + + var stderrErr error + go func() { + reader := bufio.NewReader(stderr) + // TODO CONTINUE: The formatting is messed u[] because of the new line business + // here. + // Fix that. + line, err := reader.ReadString('\n') + for err == nil { + _, err = fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", strings.TrimSpace(line)) + if err != nil { + stderrErr = err + break + } + line, err = reader.ReadString('\n') + } + + wg.Done() + }() + + wg.Wait() + + if stdoutErr != nil { + return fmt.Errorf("writing stdout failed: %w", stdoutErr) + } + + if stderrErr != nil { + return fmt.Errorf("writing stderr failed: %w", stderrErr) + } + + err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { // We don't make the parent CLI process exit with the same exit code // as the child process because the exit codes for the CLI have not @@ -108,10 +169,7 @@ Example usage: } } if err != nil { - return &runErr{ - err: err, - args: args, - } + return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) } return nil From 39d0b13c4f1a25f61a787bced2157f2717d98deb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:29:57 +0100 Subject: [PATCH 31/90] - --- libs/auth/env.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/auth/env.go b/libs/auth/env.go index ddf73c9868b..08282e46349 100644 --- a/libs/auth/env.go +++ b/libs/auth/env.go @@ -4,10 +4,6 @@ import ( "fmt" "os" "slices" -<<<<<<< HEAD - "sort" -======= ->>>>>>> origin "strings" "github.com/databricks/databricks-sdk-go/config" From 2212aa25973e6af1f62247d16ab4c08f3eac6f13 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:44:43 +0100 Subject: [PATCH 32/90] proper printing --- .../profile-is-passed/output.txt | 8 +++--- .../target-is-passed/output.txt | 12 ++++----- cmd/bundle/exec.go | 27 +++++++------------ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index 1c1b7405822..06cdd1c4400 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,12 +1,12 @@ >>> [CLI] bundle exec -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } >>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 0fce7c1a728..84a9653e928 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -1,14 +1,14 @@ >>> [CLI] bundle exec -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } >>> [CLI] bundle exec -t pat -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } >>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth @@ -19,6 +19,6 @@ Exit code: 1 >>> [CLI] bundle exec -t oauth -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index e914f88448f..4a1d7b0ebe3 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -112,37 +112,30 @@ Example usage: var stdoutErr error go func() { - reader := bufio.NewReader(stdout) - line, err := reader.ReadString('\n') - for err == nil { - _, err = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", strings.TrimSpace(line)) + defer wg.Done() + + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + _, err = fmt.Fprintln(cmd.OutOrStdout(), scanner.Text()) if err != nil { stdoutErr = err break } - line, err = reader.ReadString('\n') } - - wg.Done() }() var stderrErr error go func() { - reader := bufio.NewReader(stderr) - // TODO CONTINUE: The formatting is messed u[] because of the new line business - // here. - // Fix that. - line, err := reader.ReadString('\n') - for err == nil { - _, err = fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", strings.TrimSpace(line)) + defer wg.Done() + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + _, err = fmt.Fprintln(cmd.ErrOrStderr(), scanner.Text()) if err != nil { stderrErr = err break } - line, err = reader.ReadString('\n') } - - wg.Done() }() wg.Wait() From 7fb464b49627572d8fc1b28d8b54a3a4b6a5e624 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:47:58 +0100 Subject: [PATCH 33/90] lint --- cmd/bundle/exec.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 4a1d7b0ebe3..2bf925820df 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -22,15 +22,6 @@ func (e *exitCodeErr) Error() string { return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) } -type runErr struct { - err error - args []string -} - -func (e *runErr) Error() string { - return fmt.Sprintf("Running %q failed: %s", strings.Join(e.args, " "), e.err) -} - func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", From b5e21231eb1029f83f36575419a7df0dc54f7186 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:49:47 +0100 Subject: [PATCH 34/90] - --- acceptance/bundle/exec/basic/output.txt | 10 ++-------- acceptance/bundle/exec/basic/script | 5 +++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 9283131d6f1..c9a651aed07 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -5,13 +5,7 @@ hello, world >>> [CLI] bundle exec -- pwd [TMPDIR] ->>> [CLI] bundle exec -- echo --help ---help - ->>> [CLI] bundle exec -- bash -c exit 5 -Error: Running "bash -c exit 5" failed with exit code: 5 +>>> [CLI] bundle exec -- --help +Error: starting "--help" failed: exec: "--help": executable file not found in $PATH Exit code: 1 - ->>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr -hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index e0e238c43d2..9d1a281c2b8 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -2,8 +2,9 @@ trace $CLI bundle exec -- echo "hello, world" trace $CLI bundle exec -- pwd -# The CLI should not parse the --help flag and should pass it as is to echo. -trace $CLI bundle exec -- echo --help +# The CLI should not parse the --help flag and should try to run it as an executable +# instead. +trace $CLI bundle exec -- --help # The error message should include the exit code. errcode trace $CLI bundle exec -- bash -c "exit 5" From 39ec48a602427ecf303d845ad0bef7e0a06ea63e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:58:18 +0100 Subject: [PATCH 35/90] remove error struct --- .../databricks-cli/target-is-passed/output.txt | 2 +- cmd/bundle/exec.go | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 84a9653e928..0751ac7dc03 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -13,7 +13,7 @@ >>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: Running "[CLI] current-user me -t oauth" failed with exit code: 1 +Error: running "[CLI] current-user me -t oauth" failed with exit code: 1 Exit code: 1 diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 2bf925820df..bbb8038b95e 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -13,15 +13,6 @@ import ( "github.com/spf13/cobra" ) -type exitCodeErr struct { - exitCode int - args []string -} - -func (e *exitCodeErr) Error() string { - return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) -} - func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -147,10 +138,7 @@ Example usage: // // This keeps the door open for us to associate specific exit codes // with specific classes of errors in the future. - return &exitCodeErr{ - exitCode: exitErr.ExitCode(), - args: args, - } + return fmt.Errorf("running %q failed with exit code: %d", strings.Join(args, " "), exitErr.ExitCode()) } if err != nil { return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) From 2c773683bc3133d11fd0cf04c2ca5de0431db4a8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:08:09 +0100 Subject: [PATCH 36/90] split windows test --- acceptance/bundle/exec/basic/test.toml | 3 +++ acceptance/bundle/exec/cmd-not-found/test.toml | 3 +++ acceptance/bundle/exec/cwd/{ => unix}/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/{ => unix}/databricks.yml | 0 acceptance/bundle/exec/cwd/{ => unix}/output.txt | 0 acceptance/bundle/exec/cwd/{ => unix}/script | 0 acceptance/bundle/exec/cwd/unix/test.toml | 4 ++++ acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/windows/databricks.yml | 2 ++ acceptance/bundle/exec/cwd/windows/output.txt | 8 ++++++++ acceptance/bundle/exec/cwd/windows/script | 6 ++++++ acceptance/bundle/exec/cwd/windows/test.toml | 4 ++++ 12 files changed, 30 insertions(+) create mode 100644 acceptance/bundle/exec/basic/test.toml create mode 100644 acceptance/bundle/exec/cmd-not-found/test.toml rename acceptance/bundle/exec/cwd/{ => unix}/a/b/c/.gitkeep (100%) rename acceptance/bundle/exec/cwd/{ => unix}/databricks.yml (100%) rename acceptance/bundle/exec/cwd/{ => unix}/output.txt (100%) rename acceptance/bundle/exec/cwd/{ => unix}/script (100%) create mode 100644 acceptance/bundle/exec/cwd/unix/test.toml create mode 100644 acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep create mode 100644 acceptance/bundle/exec/cwd/windows/databricks.yml create mode 100644 acceptance/bundle/exec/cwd/windows/output.txt create mode 100644 acceptance/bundle/exec/cwd/windows/script create mode 100644 acceptance/bundle/exec/cwd/windows/test.toml diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml new file mode 100644 index 00000000000..ca7d069ae60 --- /dev/null +++ b/acceptance/bundle/exec/basic/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml new file mode 100644 index 00000000000..ca7d069ae60 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" diff --git a/acceptance/bundle/exec/cwd/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep similarity index 100% rename from acceptance/bundle/exec/cwd/a/b/c/.gitkeep rename to acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep diff --git a/acceptance/bundle/exec/cwd/databricks.yml b/acceptance/bundle/exec/cwd/unix/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cwd/databricks.yml rename to acceptance/bundle/exec/cwd/unix/databricks.yml diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/unix/output.txt similarity index 100% rename from acceptance/bundle/exec/cwd/output.txt rename to acceptance/bundle/exec/cwd/unix/output.txt diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/unix/script similarity index 100% rename from acceptance/bundle/exec/cwd/script rename to acceptance/bundle/exec/cwd/unix/script diff --git a/acceptance/bundle/exec/cwd/unix/test.toml b/acceptance/bundle/exec/cwd/unix/test.toml new file mode 100644 index 00000000000..7f30bdcde29 --- /dev/null +++ b/acceptance/bundle/exec/cwd/unix/test.toml @@ -0,0 +1,4 @@ +[GOOS] +windows = false +linux = true +darwin = true diff --git a/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/exec/cwd/windows/databricks.yml b/acceptance/bundle/exec/cwd/windows/databricks.yml new file mode 100644 index 00000000000..432311dab0d --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/cwd/windows/output.txt b/acceptance/bundle/exec/cwd/windows/output.txt new file mode 100644 index 00000000000..e6496afab82 --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/output.txt @@ -0,0 +1,8 @@ + +>>> cd a/b/c + +>>> pwd +[TMPDIR]/a/b/c + +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/cwd/windows/script b/acceptance/bundle/exec/cwd/windows/script new file mode 100644 index 00000000000..3461f6afcde --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/script @@ -0,0 +1,6 @@ +trace cd a/b/c + +trace pwd + +# Scripts that bundle exec executes should run from the bundle root. +trace $CLI bundle exec -- dir diff --git a/acceptance/bundle/exec/cwd/windows/test.toml b/acceptance/bundle/exec/cwd/windows/test.toml new file mode 100644 index 00000000000..22f2c79f0b8 --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/test.toml @@ -0,0 +1,4 @@ +[GOOS] +windows = true +linux = false +darwin = false From 4ef33dc2b4b688f306184526f042008acc13a0c1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:09:26 +0100 Subject: [PATCH 37/90] Revert "split windows test" This reverts commit 2c773683bc3133d11fd0cf04c2ca5de0431db4a8. --- acceptance/bundle/exec/basic/test.toml | 3 --- acceptance/bundle/exec/cmd-not-found/test.toml | 3 --- acceptance/bundle/exec/cwd/{unix => }/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/{unix => }/databricks.yml | 0 acceptance/bundle/exec/cwd/{unix => }/output.txt | 0 acceptance/bundle/exec/cwd/{unix => }/script | 0 acceptance/bundle/exec/cwd/unix/test.toml | 4 ---- acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/windows/databricks.yml | 2 -- acceptance/bundle/exec/cwd/windows/output.txt | 8 -------- acceptance/bundle/exec/cwd/windows/script | 6 ------ acceptance/bundle/exec/cwd/windows/test.toml | 4 ---- 12 files changed, 30 deletions(-) delete mode 100644 acceptance/bundle/exec/basic/test.toml delete mode 100644 acceptance/bundle/exec/cmd-not-found/test.toml rename acceptance/bundle/exec/cwd/{unix => }/a/b/c/.gitkeep (100%) rename acceptance/bundle/exec/cwd/{unix => }/databricks.yml (100%) rename acceptance/bundle/exec/cwd/{unix => }/output.txt (100%) rename acceptance/bundle/exec/cwd/{unix => }/script (100%) delete mode 100644 acceptance/bundle/exec/cwd/unix/test.toml delete mode 100644 acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep delete mode 100644 acceptance/bundle/exec/cwd/windows/databricks.yml delete mode 100644 acceptance/bundle/exec/cwd/windows/output.txt delete mode 100644 acceptance/bundle/exec/cwd/windows/script delete mode 100644 acceptance/bundle/exec/cwd/windows/test.toml diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml deleted file mode 100644 index ca7d069ae60..00000000000 --- a/acceptance/bundle/exec/basic/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -[[Repls]] -Old = "%PATH%" -New = "$PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml deleted file mode 100644 index ca7d069ae60..00000000000 --- a/acceptance/bundle/exec/cmd-not-found/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -[[Repls]] -Old = "%PATH%" -New = "$PATH" diff --git a/acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/a/b/c/.gitkeep similarity index 100% rename from acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep rename to acceptance/bundle/exec/cwd/a/b/c/.gitkeep diff --git a/acceptance/bundle/exec/cwd/unix/databricks.yml b/acceptance/bundle/exec/cwd/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cwd/unix/databricks.yml rename to acceptance/bundle/exec/cwd/databricks.yml diff --git a/acceptance/bundle/exec/cwd/unix/output.txt b/acceptance/bundle/exec/cwd/output.txt similarity index 100% rename from acceptance/bundle/exec/cwd/unix/output.txt rename to acceptance/bundle/exec/cwd/output.txt diff --git a/acceptance/bundle/exec/cwd/unix/script b/acceptance/bundle/exec/cwd/script similarity index 100% rename from acceptance/bundle/exec/cwd/unix/script rename to acceptance/bundle/exec/cwd/script diff --git a/acceptance/bundle/exec/cwd/unix/test.toml b/acceptance/bundle/exec/cwd/unix/test.toml deleted file mode 100644 index 7f30bdcde29..00000000000 --- a/acceptance/bundle/exec/cwd/unix/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -[GOOS] -windows = false -linux = true -darwin = true diff --git a/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/acceptance/bundle/exec/cwd/windows/databricks.yml b/acceptance/bundle/exec/cwd/windows/databricks.yml deleted file mode 100644 index 432311dab0d..00000000000 --- a/acceptance/bundle/exec/cwd/windows/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: foobar diff --git a/acceptance/bundle/exec/cwd/windows/output.txt b/acceptance/bundle/exec/cwd/windows/output.txt deleted file mode 100644 index e6496afab82..00000000000 --- a/acceptance/bundle/exec/cwd/windows/output.txt +++ /dev/null @@ -1,8 +0,0 @@ - ->>> cd a/b/c - ->>> pwd -[TMPDIR]/a/b/c - ->>> [CLI] bundle exec -- pwd -[TMPDIR] diff --git a/acceptance/bundle/exec/cwd/windows/script b/acceptance/bundle/exec/cwd/windows/script deleted file mode 100644 index 3461f6afcde..00000000000 --- a/acceptance/bundle/exec/cwd/windows/script +++ /dev/null @@ -1,6 +0,0 @@ -trace cd a/b/c - -trace pwd - -# Scripts that bundle exec executes should run from the bundle root. -trace $CLI bundle exec -- dir diff --git a/acceptance/bundle/exec/cwd/windows/test.toml b/acceptance/bundle/exec/cwd/windows/test.toml deleted file mode 100644 index 22f2c79f0b8..00000000000 --- a/acceptance/bundle/exec/cwd/windows/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -[GOOS] -windows = true -linux = false -darwin = false From 996b58a1350ee9b5defb30cfc04c0264e1e84da3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:12:05 +0100 Subject: [PATCH 38/90] fix pwd --- acceptance/bundle/exec/basic/output.txt | 3 --- acceptance/bundle/exec/basic/script | 2 -- acceptance/bundle/exec/cwd/output.txt | 4 ++-- acceptance/bundle/exec/cwd/script | 4 ++-- acceptance/bundle/exec/no-auth/output.txt | 2 +- acceptance/bundle/exec/no-auth/script | 2 +- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index c9a651aed07..4c8668232b1 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -2,9 +2,6 @@ >>> [CLI] bundle exec -- echo hello, world hello, world ->>> [CLI] bundle exec -- pwd -[TMPDIR] - >>> [CLI] bundle exec -- --help Error: starting "--help" failed: exec: "--help": executable file not found in $PATH diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 9d1a281c2b8..285023492a5 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1,7 +1,5 @@ trace $CLI bundle exec -- echo "hello, world" -trace $CLI bundle exec -- pwd - # The CLI should not parse the --help flag and should try to run it as an executable # instead. trace $CLI bundle exec -- --help diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt index e6496afab82..928bea82d87 100644 --- a/acceptance/bundle/exec/cwd/output.txt +++ b/acceptance/bundle/exec/cwd/output.txt @@ -1,8 +1,8 @@ >>> cd a/b/c ->>> pwd +>>> python3 -c import os; print(os.getcwd()) [TMPDIR]/a/b/c ->>> [CLI] bundle exec -- pwd +>>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) [TMPDIR] diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/script index 73a0966d3fd..f5e866d06af 100644 --- a/acceptance/bundle/exec/cwd/script +++ b/acceptance/bundle/exec/cwd/script @@ -1,6 +1,6 @@ trace cd a/b/c -trace pwd +trace python3 -c 'import os; print(os.getcwd())' # Scripts that bundle exec executes should run from the bundle root. -trace $CLI bundle exec -- pwd +trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt index fce8e3f613a..be94dae3616 100644 --- a/acceptance/bundle/exec/no-auth/output.txt +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -2,5 +2,5 @@ >>> [CLI] bundle exec -- echo hello hello ->>> [CLI] bundle exec -- pwd +>>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) [TMPDIR] diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index a1758ce997a..c6b9c6bfe9b 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -4,4 +4,4 @@ unset DATABRICKS_TOKEN # Confirm that bundle exec works for commands that do not require authentication, # even if authentication is not provided. trace $CLI bundle exec -- echo hello -trace $CLI bundle exec -- pwd +trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' From fcc2966118bf9aa50aa251c06d3b38aee07bcd54 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:14:08 +0100 Subject: [PATCH 39/90] - --- acceptance/bundle/exec/basic/output.txt | 8 ++++++++ acceptance/bundle/exec/basic/script | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 4c8668232b1..6056a8c9e75 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -6,3 +6,11 @@ hello, world Error: starting "--help" failed: exec: "--help": executable file not found in $PATH Exit code: 1 + +>>> [CLI] bundle exec -- bash -c exit 5 +Error: running "bash -c exit 5" failed with exit code: 5 + +Exit code: 1 + +>>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr +hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 285023492a5..20b56ad1aec 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -2,7 +2,7 @@ trace $CLI bundle exec -- echo "hello, world" # The CLI should not parse the --help flag and should try to run it as an executable # instead. -trace $CLI bundle exec -- --help +errcode trace $CLI bundle exec -- --help # The error message should include the exit code. errcode trace $CLI bundle exec -- bash -c "exit 5" From 09c05db3b18f94fa38c3440ea2d7176626bd361b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:16:40 +0100 Subject: [PATCH 40/90] - --- acceptance/bundle/exec/basic/test.toml | 3 +++ acceptance/bundle/exec/cmd-not-found/test.toml | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 acceptance/bundle/exec/basic/test.toml create mode 100644 acceptance/bundle/exec/cmd-not-found/test.toml diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml new file mode 100644 index 00000000000..ca7d069ae60 --- /dev/null +++ b/acceptance/bundle/exec/basic/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml new file mode 100644 index 00000000000..ca7d069ae60 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" From acff901127a9a592919d73fa2e19e060244ee83d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:39:09 +0100 Subject: [PATCH 41/90] fix windows --- acceptance/bundle/exec/basic/output.txt | 2 +- acceptance/bundle/exec/basic/test.toml | 6 +++++- acceptance/bundle/exec/cmd-not-found/output.txt | 2 +- acceptance/bundle/exec/cmd-not-found/script | 2 +- acceptance/bundle/exec/cmd-not-found/test.toml | 6 +++++- acceptance/bundle/exec/cwd/test.toml | 3 +++ cmd/bundle/exec.go | 2 +- 7 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/exec/cwd/test.toml diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 6056a8c9e75..ef56f139fc6 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -3,7 +3,7 @@ hello, world >>> [CLI] bundle exec -- --help -Error: starting "--help" failed: exec: "--help": executable file not found in $PATH +Error: starting "--help" failed: exec: "--help": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml index ca7d069ae60..81f2393ae4e 100644 --- a/acceptance/bundle/exec/basic/test.toml +++ b/acceptance/bundle/exec/basic/test.toml @@ -1,3 +1,7 @@ +[[Repls]] +Old = "\\$PATH" +New = "PATH" + [[Repls]] Old = "%PATH%" -New = "$PATH" +New = "PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index 8344948b44d..e6284234e0f 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ >>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH +Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/exec/cmd-not-found/script index 4a5d442f3a4..0cefaea0cd7 100644 --- a/acceptance/bundle/exec/cmd-not-found/script +++ b/acceptance/bundle/exec/cmd-not-found/script @@ -1 +1 @@ -trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 +errcode trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml index ca7d069ae60..81f2393ae4e 100644 --- a/acceptance/bundle/exec/cmd-not-found/test.toml +++ b/acceptance/bundle/exec/cmd-not-found/test.toml @@ -1,3 +1,7 @@ +[[Repls]] +Old = "\\$PATH" +New = "PATH" + [[Repls]] Old = "%PATH%" -New = "$PATH" +New = "PATH" diff --git a/acceptance/bundle/exec/cwd/test.toml b/acceptance/bundle/exec/cwd/test.toml new file mode 100644 index 00000000000..a298217f218 --- /dev/null +++ b/acceptance/bundle/exec/cwd/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '\\' +New = '/' diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index bbb8038b95e..87cc9b929e2 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -55,7 +55,7 @@ Example usage: // // This is only useful for when the Databricks CLI is the child command, // since if we do not explicitly pass the profile, the CLI will use the - // profile configured in the bundle YAML configuration (if any). + // auth configured in the bundle YAML configuration (if any). if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } From 7dc24b6484c3d6cd2827bd6e9be52d7111f61ed9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 12:56:38 +0100 Subject: [PATCH 42/90] move tests --- .../from_bundle/.databrickscfg | 3 +++ .../{ => from_bundle}/databricks.yml | 0 .../from_bundle/out.requests.txt | 9 +++++++ .../profile-is-passed/from_bundle/output.txt | 6 +++++ .../{ => from_bundle}/script | 4 ---- .../{ => from_flag}/.databrickscfg | 0 .../from_flag/databricks.yml | 5 ++++ .../{ => from_flag}/out.requests.txt | 7 +----- .../{ => from_flag}/output.txt | 6 ----- .../profile-is-passed/from_flag/script | 10 ++++++++ .../target-is-passed/default/.databrickscfg | 8 +++++++ .../{ => default}/databricks.yml | 4 +++- .../target-is-passed/default/out.requests.txt | 9 +++++++ .../target-is-passed/default/output.txt | 6 +++++ .../target-is-passed/default/script | 10 ++++++++ .../target-is-passed/from_flag/.databrickscfg | 8 +++++++ .../target-is-passed/from_flag/databricks.yml | 12 ++++++++++ .../{ => from_flag}/out.requests.txt | 16 +------------ .../target-is-passed/from_flag/output.txt | 6 +++++ .../target-is-passed/from_flag/script | 10 ++++++++ .../target-is-passed/output.txt | 24 ------------------- .../databricks-cli/target-is-passed/script | 15 ------------ 22 files changed, 107 insertions(+), 71 deletions(-) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_bundle}/databricks.yml (100%) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_bundle}/script (68%) rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_flag}/.databrickscfg (100%) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_flag}/out.requests.txt (78%) rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_flag}/output.txt (54%) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg rename acceptance/bundle/exec/databricks-cli/target-is-passed/{ => default}/databricks.yml (53%) create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/script create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml rename acceptance/bundle/exec/databricks-cli/target-is-passed/{ => from_flag}/out.requests.txt (61%) create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script delete mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt delete mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/script diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg new file mode 100644 index 00000000000..405e344a02e --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg @@ -0,0 +1,3 @@ +[someprofile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt new file mode 100644 index 00000000000..c4a1034826d --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt @@ -0,0 +1,9 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt new file mode 100644 index 00000000000..8ca3fe8c561 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -- [CLI] current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script similarity index 68% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/script rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script index 33e07494c61..a614758e5ed 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script @@ -1,5 +1,4 @@ # Replace placeholder with an actual host URL -envsubst < databricks.yml > out.yml && mv out.yml databricks.yml envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg @@ -9,6 +8,3 @@ unset DATABRICKS_TOKEN # This should use the profile configured in the bundle, i.e. PAT auth trace $CLI bundle exec -- $CLI current-user me - -# This should use myprofile, which uses oauth. -trace $CLI bundle exec --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml new file mode 100644 index 00000000000..5de7d1d9680 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: foobar + +workspace: + profile: someprofile diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt similarity index 78% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt index 2b214d4fd29..bbad66cf8bb 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt @@ -1,11 +1,6 @@ { - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" + "path": "/oidc/.well-known/oauth-authorization-server" } { "method": "GET", diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt similarity index 54% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt index 06cdd1c4400..30463a78fed 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt @@ -1,10 +1,4 @@ ->>> [CLI] bundle exec -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} - >>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { "id":"[USERID]", diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script new file mode 100644 index 00000000000..7d0124c0a60 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script @@ -0,0 +1,10 @@ +# Replace placeholder with an actual host URL +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# This should use myprofile, which uses oauth. +trace $CLI bundle exec --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg new file mode 100644 index 00000000000..dbf302e9ceb --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg @@ -0,0 +1,8 @@ +[pat-profile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[oauth-profile] +host = $DATABRICKS_HOST +client_id = client_id +client_secret = client_secret diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml similarity index 53% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml rename to acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml index 6b782c32579..7f808c24b1b 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml @@ -4,7 +4,9 @@ bundle: targets: pat: default: true + workspace: + profile: pat-profile oauth: workspace: - client_id: client_id + profile: oauth-profile diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt new file mode 100644 index 00000000000..c4a1034826d --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt @@ -0,0 +1,9 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt new file mode 100644 index 00000000000..8ca3fe8c561 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -- [CLI] current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script new file mode 100644 index 00000000000..4b036659d49 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script @@ -0,0 +1,10 @@ +# Replace placeholder with an actual host URL +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# Should use default target, which is bat based authentication +trace $CLI bundle exec -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg new file mode 100644 index 00000000000..dbf302e9ceb --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg @@ -0,0 +1,8 @@ +[pat-profile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[oauth-profile] +host = $DATABRICKS_HOST +client_id = client_id +client_secret = client_secret diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml new file mode 100644 index 00000000000..7f808c24b1b --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: foobar + +targets: + pat: + default: true + workspace: + profile: pat-profile + + oauth: + workspace: + profile: oauth-profile diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt similarity index 61% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt rename to acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt index 26efbbb08d5..bbad66cf8bb 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt @@ -1,20 +1,6 @@ { - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" + "path": "/oidc/.well-known/oauth-authorization-server" } { "method": "GET", diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt new file mode 100644 index 00000000000..828868f106b --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -t oauth -- [CLI] current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script new file mode 100644 index 00000000000..58a407a10f7 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script @@ -0,0 +1,10 @@ +# Replace placeholder with an actual host URL +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# Explicitly select oauth target +trace $CLI bundle exec -t oauth -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt deleted file mode 100644 index 0751ac7dc03..00000000000 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ /dev/null @@ -1,24 +0,0 @@ - ->>> [CLI] bundle exec -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} - ->>> [CLI] bundle exec -t pat -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} - ->>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth -Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: running "[CLI] current-user me -t oauth" failed with exit code: 1 - -Exit code: 1 - ->>> [CLI] bundle exec -t oauth -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script deleted file mode 100644 index 238c36ae4b8..00000000000 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/script +++ /dev/null @@ -1,15 +0,0 @@ -# Default target -trace $CLI bundle exec -- $CLI current-user me - -# Explicitly select default target -trace $CLI bundle exec -t pat -- $CLI current-user me - -# Conflicting targets selected. This should fail because for the child command -# pat would be configured via environment variables and oauth via the CLI resulting -# in more than one authorization method configured. -trace errcode $CLI bundle exec -t pat -- $CLI current-user me -t oauth - -# Explicitly select oauth target -unset DATABRICKS_TOKEN -export DATABRICKS_CLIENT_SECRET="client_secret" -trace $CLI bundle exec -t oauth -- $CLI current-user me From 90a95846ad0d013feead02103c70e7b06f2e9f09 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 13:00:06 +0100 Subject: [PATCH 43/90] - --- .../exec/databricks-cli/profile-is-passed/from_bundle/script | 2 +- .../exec/databricks-cli/profile-is-passed/from_flag/script | 2 +- .../exec/databricks-cli/target-is-passed/default/script | 4 ++-- .../exec/databricks-cli/target-is-passed/from_flag/script | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script index a614758e5ed..e894e458206 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script index 7d0124c0a60..9afda563b0e 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script index 4b036659d49..f719893cca1 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg @@ -6,5 +6,5 @@ export DATABRICKS_CONFIG_FILE=.databrickscfg unset DATABRICKS_HOST unset DATABRICKS_TOKEN -# Should use default target, which is bat based authentication +# Should use default target, which is pat based authentication trace $CLI bundle exec -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script index 58a407a10f7..a314d022305 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg From 8c094db50303477bde548139bf438ee7d9b89f7b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 13:17:09 +0100 Subject: [PATCH 44/90] general stderr --- acceptance/bundle/exec/basic/output.txt | 4 ++-- acceptance/bundle/exec/basic/script | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index ef56f139fc6..1b932e9b209 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -12,5 +12,5 @@ Error: running "bash -c exit 5" failed with exit code: 5 Exit code: 1 ->>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr -hello +>>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) +Hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 20b56ad1aec..1aaf140dabb 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -8,4 +8,4 @@ errcode trace $CLI bundle exec -- --help errcode trace $CLI bundle exec -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle exec -- bash -c "echo hello > /dev/stderr" +trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" From 5586121239cddfe58c21466e3185cb613aa42670 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:48:25 +0100 Subject: [PATCH 45/90] read target from flag --- cmd/bundle/exec.go | 9 +++------ cmd/root/bundle.go | 6 +++--- cmd/root/bundle_test.go | 6 +++--- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 87cc9b929e2..d781ee49371 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -7,7 +7,6 @@ import ( "strings" "sync" - "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" "github.com/spf13/cobra" @@ -42,13 +41,11 @@ Example usage: env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // If user has specified a target, pass it to the child command. DABs - // defines a "default" target which is a placeholder for when no target is defined. - // If that's the case, i.e. no targets are defined, then do not pass the target. + // If user has specified a target, pass it to the child command. // // This is only useful for when the Databricks CLI is the child command. - if b.Config.Bundle.Target != mutator.DefaultTargetPlaceholder { - env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + if target := root.GetTarget(cmd); target != "" { + env = append(env, "DATABRICKS_CONFIG_TARGET="+target) } // If the bundle has a profile configured, explicitly pass it to the child command. diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go index b4080370793..85d47431d59 100644 --- a/cmd/root/bundle.go +++ b/cmd/root/bundle.go @@ -12,8 +12,8 @@ import ( "golang.org/x/exp/maps" ) -// getTarget returns the name of the target to operate in. -func getTarget(cmd *cobra.Command) (value string) { +// GetTarget returns the name of the target to operate in. +func GetTarget(cmd *cobra.Command) (value string) { target, isFlagSet := targetFlagValue(cmd) if isFlagSet { return target @@ -77,7 +77,7 @@ func configureBundle(cmd *cobra.Command, b *bundle.Bundle) (*bundle.Bundle, diag // Load bundle and select target. ctx := cmd.Context() var diags diag.Diagnostics - if target := getTarget(cmd); target == "" { + if target := GetTarget(cmd); target == "" { diags = phases.LoadDefaultTarget(ctx, b) } else { diags = phases.LoadNamedTarget(ctx, b, target) diff --git a/cmd/root/bundle_test.go b/cmd/root/bundle_test.go index 3517b02e467..bf7c44188c5 100644 --- a/cmd/root/bundle_test.go +++ b/cmd/root/bundle_test.go @@ -215,7 +215,7 @@ func TestTargetFlagFull(t *testing.T) { err := cmd.ExecuteContext(ctx) assert.NoError(t, err) - assert.Equal(t, "development", getTarget(cmd)) + assert.Equal(t, "development", GetTarget(cmd)) } func TestTargetFlagShort(t *testing.T) { @@ -227,7 +227,7 @@ func TestTargetFlagShort(t *testing.T) { err := cmd.ExecuteContext(ctx) assert.NoError(t, err) - assert.Equal(t, "production", getTarget(cmd)) + assert.Equal(t, "production", GetTarget(cmd)) } // TODO: remove when environment flag is fully deprecated @@ -241,5 +241,5 @@ func TestTargetEnvironmentFlag(t *testing.T) { err := cmd.ExecuteContext(ctx) assert.NoError(t, err) - assert.Equal(t, "development", getTarget(cmd)) + assert.Equal(t, "development", GetTarget(cmd)) } From 26684b192e15fa6d397798483eefd3cd15d1f781 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:50:38 +0100 Subject: [PATCH 46/90] :- --- cmd/auth/describe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/auth/describe.go b/cmd/auth/describe.go index faaf64f8fd1..cf278db941f 100644 --- a/cmd/auth/describe.go +++ b/cmd/auth/describe.go @@ -57,7 +57,7 @@ func newDescribeCommand() *cobra.Command { var err error status, err = getAuthStatus(cmd, args, showSensitive, func(cmd *cobra.Command, args []string) (*config.Config, bool, error) { isAccount, err := root.MustAnyClient(cmd, args) - return root.ConfigUsed(cmd.Context()), isAccount, err + return command.ConfigUsed(cmd.Context()), isAccount, err }) if err != nil { return err From d9b5f5e18565257a3ac9655e0d12c2a794c07347 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:51:01 +0100 Subject: [PATCH 47/90] fix build --- cmd/bundle/exec.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index d781ee49371..023ea339582 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/command" "github.com/spf13/cobra" ) @@ -39,7 +40,7 @@ Example usage: childCmd := exec.Command(args[0], args[1:]...) - env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + env := auth.ProcessEnv(command.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. // From 987220b9de6321fdbe70244a0e337341540a7dd4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:54:20 +0100 Subject: [PATCH 48/90] update docs --- cmd/bundle/exec.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 023ea339582..3a04d6a984a 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -20,9 +20,12 @@ func newExecCommand() *cobra.Command { Args: cobra.MinimumNArgs(1), Long: `Execute a command using the same authentication context as the bundle -The current working directory of the provided command will be set to the root +Note: The current working directory of the provided command will be set to the root of the bundle. +Authentication to the input command will be provided by setting the appropriate +environment variables that Databricks tools use to authenticate. + Example usage: 1. databricks bundle exec -- echo "hello, world" 2. databricks bundle exec -- /bin/bash -c "echo hello" From 98f33a80b8b52deaedf6bb4610149093fee0536e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 14:28:45 +0200 Subject: [PATCH 49/90] - --- cmd/bundle/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index b7683a676c6..574ad101632 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -46,7 +46,7 @@ func resolveRunArgument(ctx context.Context, b *bundle.Bundle, args []string) (s return "", nil, err } return key, args, nil - } a + } if len(args) < 1 { return "", nil, errors.New("expected a KEY of the resource to run") From 9ea61443603ff4ed771f0deb3aa72f7b2b814791 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 16:19:46 +0200 Subject: [PATCH 50/90] use execve --- acceptance/bundle/exec/basic/output.txt | 5 +- .../bundle/exec/cmd-not-found/output.txt | 2 +- acceptance/bundle/exec/cwd/output.txt | 4 +- acceptance/bundle/exec/no-auth/output.txt | 2 +- cmd/bundle/exec.go | 97 +++---------------- libs/exec/execv.go | 22 +++++ libs/exec/execv_unix.go | 24 +++++ libs/exec/execv_windows.go | 95 ++++++++++++++++++ 8 files changed, 158 insertions(+), 93 deletions(-) create mode 100644 libs/exec/execv.go create mode 100644 libs/exec/execv_unix.go create mode 100644 libs/exec/execv_windows.go diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 1b932e9b209..9cb5f8c7b8f 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -3,14 +3,13 @@ hello, world >>> [CLI] bundle exec -- --help -Error: starting "--help" failed: exec: "--help": executable file not found in PATH +Error: running "--help" failed: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH Exit code: 1 >>> [CLI] bundle exec -- bash -c exit 5 -Error: running "bash -c exit 5" failed with exit code: 5 -Exit code: 1 +Exit code: 5 >>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index e6284234e0f..73bc70e6c65 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ >>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in PATH +Error: running "doesnotexist arg1 arg2 --flag1 --flag2" failed: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt index 928bea82d87..9276774605a 100644 --- a/acceptance/bundle/exec/cwd/output.txt +++ b/acceptance/bundle/exec/cwd/output.txt @@ -2,7 +2,7 @@ >>> cd a/b/c >>> python3 -c import os; print(os.getcwd()) -[TMPDIR]/a/b/c +[TEST_TMP_DIR]/a/b/c >>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) -[TMPDIR] +[TEST_TMP_DIR] diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt index be94dae3616..436991346b3 100644 --- a/acceptance/bundle/exec/no-auth/output.txt +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -3,4 +3,4 @@ hello >>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) -[TMPDIR] +[TEST_TMP_DIR] diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 3a04d6a984a..f6473cc2eb7 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -1,15 +1,13 @@ package bundle import ( - "bufio" "fmt" - "os/exec" "strings" - "sync" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" - "github.com/databricks/cli/libs/command" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/exec" "github.com/spf13/cobra" ) @@ -41,9 +39,7 @@ Example usage: return diags.Error() } - childCmd := exec.Command(args[0], args[1:]...) - - env := auth.ProcessEnv(command.ConfigUsed(cmd.Context())) + env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. // @@ -61,86 +57,15 @@ Example usage: env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } - childCmd.Env = env - - // Execute all scripts from the bundle root directory. This behavior can - // be surprising in isolation, but we do it to keep the behavior consistent - // for both these cases: - // 1. One shot commands like `databricks bundle exec -- echo hello` - // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. - // - // TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable - // that users can read to figure out the original CWD. I'll do that when - // adding support for the scripts section. - childCmd.Dir = b.BundleRootPath - - stdout, err := childCmd.StdoutPipe() - if err != nil { - return fmt.Errorf("creating stdout pipe failed: %w", err) - } - - stderr, err := childCmd.StderrPipe() - if err != nil { - return fmt.Errorf("creating stderr pipe failed: %w", err) - } - - // Start the child command. - err = childCmd.Start() - if err != nil { - return fmt.Errorf("starting %q failed: %w", strings.Join(args, " "), err) - } + // TODO: Test singal propogation in the acceptance testS? - var wg sync.WaitGroup - wg.Add(2) - - var stdoutErr error - go func() { - defer wg.Done() - - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - _, err = fmt.Fprintln(cmd.OutOrStdout(), scanner.Text()) - if err != nil { - stdoutErr = err - break - } - } - }() - - var stderrErr error - go func() { - defer wg.Done() - - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - _, err = fmt.Fprintln(cmd.ErrOrStderr(), scanner.Text()) - if err != nil { - stderrErr = err - break - } - } - }() - - wg.Wait() - - if stdoutErr != nil { - return fmt.Errorf("writing stdout failed: %w", stdoutErr) - } - - if stderrErr != nil { - return fmt.Errorf("writing stderr failed: %w", stderrErr) - } - - err = childCmd.Wait() - if exitErr, ok := err.(*exec.ExitError); ok { - // We don't make the parent CLI process exit with the same exit code - // as the child process because the exit codes for the CLI have not - // been standardized yet. - // - // This keeps the door open for us to associate specific exit codes - // with specific classes of errors in the future. - return fmt.Errorf("running %q failed with exit code: %d", strings.Join(args, " "), exitErr.ExitCode()) - } + err := exec.Execv(exec.ExecvOptions{ + Args: args, + Env: env, + Dir: b.BundleRootPath, + Stderr: cmd.ErrOrStderr(), + Stdout: cmd.OutOrStdout(), + }) if err != nil { return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) } diff --git a/libs/exec/execv.go b/libs/exec/execv.go new file mode 100644 index 00000000000..c477d76dd40 --- /dev/null +++ b/libs/exec/execv.go @@ -0,0 +1,22 @@ +package exec + +import "io" + +// TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable +// that users can read to figure out the original CWD. I'll do that when +// adding support for the scripts section. +type ExecvOptions struct { + Args []string + Env []string + Dir string + + // Stderr and Stdout are only used for Windows where execv is not supported. + // For Unix, the Stdout and Stderr are automatically inherited during the exec + // system call. + Stderr io.Writer + Stdout io.Writer +} + +func Execv(opts ExecvOptions) error { + return execv(opts) +} diff --git a/libs/exec/execv_unix.go b/libs/exec/execv_unix.go new file mode 100644 index 00000000000..663cc1360c5 --- /dev/null +++ b/libs/exec/execv_unix.go @@ -0,0 +1,24 @@ +//go:build linux || darwin + +package exec + +import ( + "fmt" + "os" + "os/exec" + "syscall" +) + +func execv(opts ExecvOptions) error { + err := os.Chdir(opts.Dir) + if err != nil { + return fmt.Errorf("changing directory to %s failed: %w", opts.Dir, err) + } + + // execve syscall does not perform PATH lookup and + path, err := exec.LookPath(opts.Args[0]) + if err != nil { + return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + } + return syscall.Exec(path, opts.Args, opts.Env) +} diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go new file mode 100644 index 00000000000..04be6edf98e --- /dev/null +++ b/libs/exec/execv_windows.go @@ -0,0 +1,95 @@ +//go:build windows + +package exec + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "strings" + "sync" +) + +// Note: Windows does not support an execv syscall that replaces the current process. +// Thus we emulate that by launching a child process and streaming the output and returning +// the exit code. +// ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/execv-wexecv?view=msvc-170 +func execv(opts ExecvOptions) error { + cmd := exec.Command(opts.Name, opts.Args...) + + // TODO: Move this comment. + // Execute all scripts from the bundle root directory. This behavior can + // be surprising in isolation, but we do it to keep the behavior consistent + // for both these cases: + // 1. One shot commands like `databricks bundle exec -- echo hello` + // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. + cmd.Dir = opts.Dir + + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("creating stdout pipe failed: %w", err) + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("creating stderr pipe failed: %w", err) + } + + // Start the child command. + err = cmd.Start() + if err != nil { + return fmt.Errorf("starting %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + } + + var wg sync.WaitGroup + wg.Add(2) + + var stdoutErr error + go func() { + defer wg.Done() + + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + _, err = fmt.Fprintln(opts.Stdout, scanner.Text()) + if err != nil { + stdoutErr = err + break + } + } + }() + + var stderrErr error + go func() { + defer wg.Done() + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + _, err = fmt.Fprintln(opts.Stderr, scanner.Text()) + if err != nil { + stderrErr = err + break + } + } + }() + + wg.Wait() + + if stdoutErr != nil { + return fmt.Errorf("writing stdout failed: %w", stdoutErr) + } + + if stderrErr != nil { + return fmt.Errorf("writing stderr failed: %w", stderrErr) + } + + err = cmd.Wait() + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + if err != nil { + return fmt.Errorf("running %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + } + + return nil +} From 3c4ff15e007ce05c253c5ba5293ec6c38bb206c2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:31:10 +0200 Subject: [PATCH 51/90] switch over to run --- acceptance/bundle/exec/basic/output.txt | 10 ++--- acceptance/bundle/exec/basic/script | 8 ++-- .../bundle/exec/cmd-not-found/output.txt | 4 +- acceptance/bundle/exec/cmd-not-found/script | 2 +- acceptance/bundle/exec/cwd/output.txt | 2 +- acceptance/bundle/exec/cwd/script | 4 +- .../profile-is-passed/from_bundle/output.txt | 2 +- .../profile-is-passed/from_bundle/script | 2 +- .../profile-is-passed/from_flag/output.txt | 2 +- .../profile-is-passed/from_flag/script | 2 +- .../target-is-passed/default/output.txt | 2 +- .../target-is-passed/default/script | 2 +- .../target-is-passed/from_flag/output.txt | 2 +- .../target-is-passed/from_flag/script | 2 +- acceptance/bundle/exec/no-auth/output.txt | 4 +- acceptance/bundle/exec/no-auth/script | 6 +-- acceptance/bundle/exec/no-bundle/output.txt | 2 +- acceptance/bundle/exec/no-bundle/script | 2 +- .../bundle/exec/no-separator/output.txt | 4 +- acceptance/bundle/exec/no-separator/script | 2 +- cmd/bundle/exec.go | 1 - cmd/bundle/run.go | 39 +++++++++++++++++++ 22 files changed, 72 insertions(+), 34 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 9cb5f8c7b8f..f941077c9d1 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -1,15 +1,15 @@ ->>> [CLI] bundle exec -- echo hello, world +>>> [CLI] bundle run -- echo hello, world hello, world ->>> [CLI] bundle exec -- --help -Error: running "--help" failed: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH +>>> [CLI] bundle run -- --help +Error: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH Exit code: 1 ->>> [CLI] bundle exec -- bash -c exit 5 +>>> [CLI] bundle run -- bash -c exit 5 Exit code: 5 ->>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) +>>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 1aaf140dabb..8bcb80cc416 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1,11 +1,11 @@ -trace $CLI bundle exec -- echo "hello, world" +trace $CLI bundle run -- echo "hello, world" # The CLI should not parse the --help flag and should try to run it as an executable # instead. -errcode trace $CLI bundle exec -- --help +errcode trace $CLI bundle run -- --help # The error message should include the exit code. -errcode trace $CLI bundle exec -- bash -c "exit 5" +errcode trace $CLI bundle run -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" +trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index 73bc70e6c65..275f67bf714 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: running "doesnotexist arg1 arg2 --flag1 --flag2" failed: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH +>>> [CLI] bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 +Error: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/exec/cmd-not-found/script index 0cefaea0cd7..78919d576e4 100644 --- a/acceptance/bundle/exec/cmd-not-found/script +++ b/acceptance/bundle/exec/cmd-not-found/script @@ -1 +1 @@ -errcode trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 +errcode trace $CLI bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt index 9276774605a..8a571d5de87 100644 --- a/acceptance/bundle/exec/cwd/output.txt +++ b/acceptance/bundle/exec/cwd/output.txt @@ -4,5 +4,5 @@ >>> python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR]/a/b/c ->>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) +>>> [CLI] bundle run -- python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR] diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/script index f5e866d06af..b1064252442 100644 --- a/acceptance/bundle/exec/cwd/script +++ b/acceptance/bundle/exec/cwd/script @@ -2,5 +2,5 @@ trace cd a/b/c trace python3 -c 'import os; print(os.getcwd())' -# Scripts that bundle exec executes should run from the bundle root. -trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' +# Scripts that bundle run executes should run from the bundle root. +trace $CLI bundle run -- python3 -c 'import os; print(os.getcwd())' diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt index 8ca3fe8c561..704a6ac915e 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- [CLI] current-user me +>>> [CLI] bundle run -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script index e894e458206..28bc0f1b84e 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # This should use the profile configured in the bundle, i.e. PAT auth -trace $CLI bundle exec -- $CLI current-user me +trace $CLI bundle run -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt index 30463a78fed..e2c200ce1de 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me +>>> [CLI] bundle run --profile myprofile -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script index 9afda563b0e..41049c25f48 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # This should use myprofile, which uses oauth. -trace $CLI bundle exec --profile myprofile -- $CLI current-user me +trace $CLI bundle run --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt index 8ca3fe8c561..704a6ac915e 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- [CLI] current-user me +>>> [CLI] bundle run -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script index f719893cca1..f9d3610b4e0 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # Should use default target, which is pat based authentication -trace $CLI bundle exec -- $CLI current-user me +trace $CLI bundle run -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt index 828868f106b..9d568529fda 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -t oauth -- [CLI] current-user me +>>> [CLI] bundle run -t oauth -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script index a314d022305..fcae775022c 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # Explicitly select oauth target -trace $CLI bundle exec -t oauth -- $CLI current-user me +trace $CLI bundle run -t oauth -- $CLI current-user me diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt index 436991346b3..62570732824 100644 --- a/acceptance/bundle/exec/no-auth/output.txt +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -1,6 +1,6 @@ ->>> [CLI] bundle exec -- echo hello +>>> [CLI] bundle run -- echo hello hello ->>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) +>>> [CLI] bundle run -- python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR] diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index c6b9c6bfe9b..f96de1cf0b9 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -1,7 +1,7 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN -# Confirm that bundle exec works for commands that do not require authentication, +# Confirm that bundle run works for commands that do not require authentication, # even if authentication is not provided. -trace $CLI bundle exec -- echo hello -trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' +trace $CLI bundle run -- echo hello +trace $CLI bundle run -- python3 -c 'import os; print(os.getcwd())' diff --git a/acceptance/bundle/exec/no-bundle/output.txt b/acceptance/bundle/exec/no-bundle/output.txt index 7030c5405ce..f961335f9d4 100644 --- a/acceptance/bundle/exec/no-bundle/output.txt +++ b/acceptance/bundle/exec/no-bundle/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- echo hello +>>> [CLI] bundle run -- echo hello Error: unable to locate bundle root: databricks.yml not found Exit code: 1 diff --git a/acceptance/bundle/exec/no-bundle/script b/acceptance/bundle/exec/no-bundle/script index b872f65db1f..925f1c19cda 100644 --- a/acceptance/bundle/exec/no-bundle/script +++ b/acceptance/bundle/exec/no-bundle/script @@ -1 +1 @@ -trace $CLI bundle exec -- echo hello +trace $CLI bundle run -- echo hello diff --git a/acceptance/bundle/exec/no-separator/output.txt b/acceptance/bundle/exec/no-separator/output.txt index ef7b6f1ce12..693c5d324ac 100644 --- a/acceptance/bundle/exec/no-separator/output.txt +++ b/acceptance/bundle/exec/no-separator/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec echo hello -Error: Please add a '--' separator. Usage: 'databricks bundle exec -- echo hello' +>>> [CLI] bundle run echo hello +Error: unable to locate bundle root: databricks.yml not found Exit code: 1 diff --git a/acceptance/bundle/exec/no-separator/script b/acceptance/bundle/exec/no-separator/script index 050538fa4d0..6a0dfbf29a5 100644 --- a/acceptance/bundle/exec/no-separator/script +++ b/acceptance/bundle/exec/no-separator/script @@ -1 +1 @@ -trace $CLI bundle exec echo hello +trace $CLI bundle run echo hello diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index f6473cc2eb7..c0f5f804f08 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -57,7 +57,6 @@ Example usage: env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } - // TODO: Test singal propogation in the acceptance testS? err := exec.Execv(exec.ExecvOptions{ Args: args, diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 574ad101632..3fed411e8e2 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -14,7 +14,10 @@ import ( "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/exec" "github.com/databricks/cli/libs/flags" "github.com/spf13/cobra" "golang.org/x/exp/maps" @@ -111,6 +114,13 @@ task or a Python wheel task, the second example applies. return diags.Error() } + // If user runs the bundle run command as: + // databricks bundle run -- + // we execute the command inline. + if cmd.ArgsLenAtDash() == 0 { + return executeInline(cmd, args, b) + } + diags = phases.Initialize(ctx, b) if err := diags.Error(); err != nil { return err @@ -209,3 +219,32 @@ task or a Python wheel task, the second example applies. return cmd } + +// TODO: Test singal propogation in the acceptance testS? +func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { + env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) + + // If user has specified a target, pass it to the child command. + // + // This is only useful for when the Databricks CLI is the child command. + if target := root.GetTarget(cmd); target != "" { + env = append(env, "DATABRICKS_CONFIG_TARGET="+target) + } + + // If the bundle has a profile configured, explicitly pass it to the child command. + // + // This is only useful for when the Databricks CLI is the child command, + // since if we do not explicitly pass the profile, the CLI will use the + // auth configured in the bundle YAML configuration (if any). + if b.Config.Workspace.Profile != "" { + env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) + } + + return exec.Execv(exec.ExecvOptions{ + Args: args, + Env: env, + Dir: b.BundleRootPath, + Stderr: cmd.ErrOrStderr(), + Stdout: cmd.OutOrStdout(), + }) +} From 2866956ba74adb286443502fd27360646bbc07ac Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:43:48 +0200 Subject: [PATCH 52/90] -gs --- libs/exec/{execv_windows.go => exec_windows.go} | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) rename libs/exec/{execv_windows.go => exec_windows.go} (82%) diff --git a/libs/exec/execv_windows.go b/libs/exec/exec_windows.go similarity index 82% rename from libs/exec/execv_windows.go rename to libs/exec/exec_windows.go index 04be6edf98e..15a29d842d3 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/exec_windows.go @@ -1,5 +1,4 @@ //go:build windows - package exec import ( @@ -16,7 +15,13 @@ import ( // the exit code. // ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/execv-wexecv?view=msvc-170 func execv(opts ExecvOptions) error { - cmd := exec.Command(opts.Name, opts.Args...) + path, err := exec.LookPath(opts.Args[0]) + if err != nil { + return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + } + + // TODO: Validate atleast one arg before calling execv. + cmd := exec.Command(path, opts.Args[1:]...) // TODO: Move this comment. // Execute all scripts from the bundle root directory. This behavior can @@ -25,6 +30,7 @@ func execv(opts ExecvOptions) error { // 1. One shot commands like `databricks bundle exec -- echo hello` // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. cmd.Dir = opts.Dir + cmd.Env = opts.Env stdout, err := cmd.StdoutPipe() if err != nil { @@ -39,7 +45,7 @@ func execv(opts ExecvOptions) error { // Start the child command. err = cmd.Start() if err != nil { - return fmt.Errorf("starting %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + return fmt.Errorf(" %s failed: %w", strings.Join(opts.Args, " "), err) } var wg sync.WaitGroup @@ -88,7 +94,7 @@ func execv(opts ExecvOptions) error { os.Exit(exitErr.ExitCode()) } if err != nil { - return fmt.Errorf("running %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + return fmt.Errorf("running %s failed: %w", strings.Join(opts.Args, " "), err) } return nil From 6239f4b952366aeb2ef2b81d546ac1528b6515f1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:43:56 +0200 Subject: [PATCH 53/90] - --- libs/exec/{exec_windows.go => execv_windows.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename libs/exec/{exec_windows.go => execv_windows.go} (100%) diff --git a/libs/exec/exec_windows.go b/libs/exec/execv_windows.go similarity index 100% rename from libs/exec/exec_windows.go rename to libs/exec/execv_windows.go From 28ea87e1639f004f5ee8a1e3a8a7c84e25bd32c5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:48:58 +0200 Subject: [PATCH 54/90] - --- .../bundle/{exec => run/inline-script}/basic/databricks.yml | 0 .../bundle/{exec => run/inline-script}/basic/output.txt | 4 +++- acceptance/bundle/{exec => run/inline-script}/basic/script | 0 acceptance/bundle/{exec => run/inline-script}/basic/test.toml | 0 .../{exec => run/inline-script}/cmd-not-found/databricks.yml | 0 .../{exec => run/inline-script}/cmd-not-found/output.txt | 0 .../bundle/{exec => run/inline-script}/cmd-not-found/script | 0 .../{exec => run/inline-script}/cmd-not-found/test.toml | 0 .../bundle/{exec => run/inline-script}/cwd/a/b/c/.gitkeep | 0 .../bundle/{exec => run/inline-script}/cwd/databricks.yml | 0 acceptance/bundle/{exec => run/inline-script}/cwd/output.txt | 0 acceptance/bundle/{exec => run/inline-script}/cwd/script | 0 acceptance/bundle/{exec => run/inline-script}/cwd/test.toml | 0 .../profile-is-passed/from_bundle/.databrickscfg | 0 .../profile-is-passed/from_bundle/databricks.yml | 0 .../profile-is-passed/from_bundle/out.requests.txt | 0 .../databricks-cli/profile-is-passed/from_bundle/output.txt | 0 .../databricks-cli/profile-is-passed/from_bundle/script | 0 .../databricks-cli/profile-is-passed/from_flag/.databrickscfg | 0 .../databricks-cli/profile-is-passed/from_flag/databricks.yml | 0 .../profile-is-passed/from_flag/out.requests.txt | 0 .../databricks-cli/profile-is-passed/from_flag/output.txt | 0 .../databricks-cli/profile-is-passed/from_flag/script | 0 .../databricks-cli/target-is-passed/default/.databrickscfg | 0 .../databricks-cli/target-is-passed/default/databricks.yml | 0 .../databricks-cli/target-is-passed/default/out.requests.txt | 0 .../databricks-cli/target-is-passed/default/output.txt | 0 .../databricks-cli/target-is-passed/default/script | 0 .../databricks-cli/target-is-passed/from_flag/.databrickscfg | 0 .../databricks-cli/target-is-passed/from_flag/databricks.yml | 0 .../target-is-passed/from_flag/out.requests.txt | 0 .../databricks-cli/target-is-passed/from_flag/output.txt | 0 .../databricks-cli/target-is-passed/from_flag/script | 0 .../{exec => run/inline-script}/databricks-cli/test.toml | 0 .../bundle/{exec => run/inline-script}/no-auth/databricks.yml | 0 .../bundle/{exec => run/inline-script}/no-auth/output.txt | 0 acceptance/bundle/{exec => run/inline-script}/no-auth/script | 0 .../bundle/{exec => run/inline-script}/no-bundle/output.txt | 0 .../bundle/{exec => run/inline-script}/no-bundle/script | 0 .../{exec => run/inline-script}/no-separator/output.txt | 0 .../bundle/{exec => run/inline-script}/no-separator/script | 0 acceptance/bundle/{exec => run/inline-script}/test.toml | 0 cmd/bundle/run.go | 2 +- 43 files changed, 4 insertions(+), 2 deletions(-) rename acceptance/bundle/{exec => run/inline-script}/basic/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/basic/output.txt (71%) rename acceptance/bundle/{exec => run/inline-script}/basic/script (100%) rename acceptance/bundle/{exec => run/inline-script}/basic/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/script (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/a/b/c/.gitkeep (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/script (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/no-auth/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/no-auth/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/no-auth/script (100%) rename acceptance/bundle/{exec => run/inline-script}/no-bundle/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/no-bundle/script (100%) rename acceptance/bundle/{exec => run/inline-script}/no-separator/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/no-separator/script (100%) rename acceptance/bundle/{exec => run/inline-script}/test.toml (100%) diff --git a/acceptance/bundle/exec/basic/databricks.yml b/acceptance/bundle/run/inline-script/basic/databricks.yml similarity index 100% rename from acceptance/bundle/exec/basic/databricks.yml rename to acceptance/bundle/run/inline-script/basic/databricks.yml diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt similarity index 71% rename from acceptance/bundle/exec/basic/output.txt rename to acceptance/bundle/run/inline-script/basic/output.txt index f941077c9d1..b26b3f32320 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -11,5 +11,7 @@ Exit code: 1 Exit code: 5 ->>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) +>>> cat stderr.txt + +>>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/run/inline-script/basic/script similarity index 100% rename from acceptance/bundle/exec/basic/script rename to acceptance/bundle/run/inline-script/basic/script diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/run/inline-script/basic/test.toml similarity index 100% rename from acceptance/bundle/exec/basic/test.toml rename to acceptance/bundle/run/inline-script/basic/test.toml diff --git a/acceptance/bundle/exec/cmd-not-found/databricks.yml b/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/databricks.yml rename to acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/run/inline-script/cmd-not-found/output.txt similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/output.txt rename to acceptance/bundle/run/inline-script/cmd-not-found/output.txt diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/run/inline-script/cmd-not-found/script similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/script rename to acceptance/bundle/run/inline-script/cmd-not-found/script diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/run/inline-script/cmd-not-found/test.toml similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/test.toml rename to acceptance/bundle/run/inline-script/cmd-not-found/test.toml diff --git a/acceptance/bundle/exec/cwd/a/b/c/.gitkeep b/acceptance/bundle/run/inline-script/cwd/a/b/c/.gitkeep similarity index 100% rename from acceptance/bundle/exec/cwd/a/b/c/.gitkeep rename to acceptance/bundle/run/inline-script/cwd/a/b/c/.gitkeep diff --git a/acceptance/bundle/exec/cwd/databricks.yml b/acceptance/bundle/run/inline-script/cwd/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cwd/databricks.yml rename to acceptance/bundle/run/inline-script/cwd/databricks.yml diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/run/inline-script/cwd/output.txt similarity index 100% rename from acceptance/bundle/exec/cwd/output.txt rename to acceptance/bundle/run/inline-script/cwd/output.txt diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/run/inline-script/cwd/script similarity index 100% rename from acceptance/bundle/exec/cwd/script rename to acceptance/bundle/run/inline-script/cwd/script diff --git a/acceptance/bundle/exec/cwd/test.toml b/acceptance/bundle/run/inline-script/cwd/test.toml similarity index 100% rename from acceptance/bundle/exec/cwd/test.toml rename to acceptance/bundle/run/inline-script/cwd/test.toml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/script diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/script rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script diff --git a/acceptance/bundle/exec/databricks-cli/test.toml b/acceptance/bundle/run/inline-script/databricks-cli/test.toml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/test.toml rename to acceptance/bundle/run/inline-script/databricks-cli/test.toml diff --git a/acceptance/bundle/exec/no-auth/databricks.yml b/acceptance/bundle/run/inline-script/no-auth/databricks.yml similarity index 100% rename from acceptance/bundle/exec/no-auth/databricks.yml rename to acceptance/bundle/run/inline-script/no-auth/databricks.yml diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/run/inline-script/no-auth/output.txt similarity index 100% rename from acceptance/bundle/exec/no-auth/output.txt rename to acceptance/bundle/run/inline-script/no-auth/output.txt diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/run/inline-script/no-auth/script similarity index 100% rename from acceptance/bundle/exec/no-auth/script rename to acceptance/bundle/run/inline-script/no-auth/script diff --git a/acceptance/bundle/exec/no-bundle/output.txt b/acceptance/bundle/run/inline-script/no-bundle/output.txt similarity index 100% rename from acceptance/bundle/exec/no-bundle/output.txt rename to acceptance/bundle/run/inline-script/no-bundle/output.txt diff --git a/acceptance/bundle/exec/no-bundle/script b/acceptance/bundle/run/inline-script/no-bundle/script similarity index 100% rename from acceptance/bundle/exec/no-bundle/script rename to acceptance/bundle/run/inline-script/no-bundle/script diff --git a/acceptance/bundle/exec/no-separator/output.txt b/acceptance/bundle/run/inline-script/no-separator/output.txt similarity index 100% rename from acceptance/bundle/exec/no-separator/output.txt rename to acceptance/bundle/run/inline-script/no-separator/output.txt diff --git a/acceptance/bundle/exec/no-separator/script b/acceptance/bundle/run/inline-script/no-separator/script similarity index 100% rename from acceptance/bundle/exec/no-separator/script rename to acceptance/bundle/run/inline-script/no-separator/script diff --git a/acceptance/bundle/exec/test.toml b/acceptance/bundle/run/inline-script/test.toml similarity index 100% rename from acceptance/bundle/exec/test.toml rename to acceptance/bundle/run/inline-script/test.toml diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 3fed411e8e2..15c01c18c8f 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -117,7 +117,7 @@ task or a Python wheel task, the second example applies. // If user runs the bundle run command as: // databricks bundle run -- // we execute the command inline. - if cmd.ArgsLenAtDash() == 0 { + if cmd.ArgsLenAtDash() == 0 && len(args) > 0 { return executeInline(cmd, args, b) } From a6d161a1b0798609e1cee2b3178ee6e186a9ff9b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:49:27 +0200 Subject: [PATCH 55/90] - --- acceptance/bundle/run/inline-script/basic/script | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index 8bcb80cc416..350705035c4 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -8,4 +8,6 @@ errcode trace $CLI bundle run -- --help errcode trace $CLI bundle run -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" +trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt +trace cat stderr.txt +rm stderr.txt From a7ffe5db820c32149565e24735f66ec48439fc8e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:51:44 +0200 Subject: [PATCH 56/90] - --- acceptance/bundle/run/inline-script/basic/output.txt | 2 +- .../bundle/run/inline-script/cmd-not-found/databricks.yml | 2 -- .../bundle/run/inline-script/cmd-not-found/output.txt | 5 ----- acceptance/bundle/run/inline-script/cmd-not-found/script | 1 - .../bundle/run/inline-script/cmd-not-found/test.toml | 7 ------- libs/exec/execv_unix.go | 2 +- libs/exec/execv_windows.go | 2 +- 7 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/output.txt delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/script delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/test.toml diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index b26b3f32320..70ffeb1f0c6 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -3,7 +3,7 @@ hello, world >>> [CLI] bundle run -- --help -Error: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH +Error: looking up "--help" failed: exec: "--help": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml b/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml deleted file mode 100644 index 432311dab0d..00000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: foobar diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/output.txt b/acceptance/bundle/run/inline-script/cmd-not-found/output.txt deleted file mode 100644 index 275f67bf714..00000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/output.txt +++ /dev/null @@ -1,5 +0,0 @@ - ->>> [CLI] bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH - -Exit code: 1 diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/script b/acceptance/bundle/run/inline-script/cmd-not-found/script deleted file mode 100644 index 78919d576e4..00000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/script +++ /dev/null @@ -1 +0,0 @@ -errcode trace $CLI bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/test.toml b/acceptance/bundle/run/inline-script/cmd-not-found/test.toml deleted file mode 100644 index 81f2393ae4e..00000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/test.toml +++ /dev/null @@ -1,7 +0,0 @@ -[[Repls]] -Old = "\\$PATH" -New = "PATH" - -[[Repls]] -Old = "%PATH%" -New = "PATH" diff --git a/libs/exec/execv_unix.go b/libs/exec/execv_unix.go index 663cc1360c5..bbb896ad037 100644 --- a/libs/exec/execv_unix.go +++ b/libs/exec/execv_unix.go @@ -18,7 +18,7 @@ func execv(opts ExecvOptions) error { // execve syscall does not perform PATH lookup and path, err := exec.LookPath(opts.Args[0]) if err != nil { - return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) } return syscall.Exec(path, opts.Args, opts.Env) } diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 15a29d842d3..a2738c67ca3 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -17,7 +17,7 @@ import ( func execv(opts ExecvOptions) error { path, err := exec.LookPath(opts.Args[0]) if err != nil { - return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) } // TODO: Validate atleast one arg before calling execv. From c56f6f5da7d2508108f8dea99076e48c380f8da7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:57:37 +0200 Subject: [PATCH 57/90] - --- acceptance/bundle/run/inline-script/no-auth/output.txt | 5 +++++ acceptance/bundle/run/inline-script/no-auth/script | 1 + 2 files changed, 6 insertions(+) diff --git a/acceptance/bundle/run/inline-script/no-auth/output.txt b/acceptance/bundle/run/inline-script/no-auth/output.txt index 62570732824..1fab88ec254 100644 --- a/acceptance/bundle/run/inline-script/no-auth/output.txt +++ b/acceptance/bundle/run/inline-script/no-auth/output.txt @@ -4,3 +4,8 @@ hello >>> [CLI] bundle run -- python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR] + +>>> [CLI] bundle run -- [CLI] current-user me +Error: default auth: cannot configure default credentials, please check https://docs.databricks.com/en/dev-tools/auth.html#databricks-client-unified-authentication to configure credentials for your preferred authentication method. Config: databricks_cli_path=[CLI]. Env: DATABRICKS_CLI_PATH + +Exit code: 1 diff --git a/acceptance/bundle/run/inline-script/no-auth/script b/acceptance/bundle/run/inline-script/no-auth/script index f96de1cf0b9..8fec852ddcc 100644 --- a/acceptance/bundle/run/inline-script/no-auth/script +++ b/acceptance/bundle/run/inline-script/no-auth/script @@ -5,3 +5,4 @@ unset DATABRICKS_TOKEN # even if authentication is not provided. trace $CLI bundle run -- echo hello trace $CLI bundle run -- python3 -c 'import os; print(os.getcwd())' +trace $CLI bundle run -- $CLI current-user me From a0fcc5d52e2c8940ff64a4f1d296fcc42d030d01 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:59:33 +0200 Subject: [PATCH 58/90] - --- bundle/config/mutator/default_target.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index c2e9e8acf27..73d99002a06 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -13,13 +13,11 @@ type defineDefaultTarget struct { name string } -const DefaultTargetPlaceholder = "default" - // DefineDefaultTarget adds a target named "default" // to the configuration if none have been defined. func DefineDefaultTarget() bundle.Mutator { return &defineDefaultTarget{ - name: DefaultTargetPlaceholder, + name: "default", } } From f30dffa1bf13ad7001e463ce5671e1deed1f6e95 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:02:01 +0200 Subject: [PATCH 59/90] - --- cmd/bundle/exec.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index c0f5f804f08..fc950157631 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -57,7 +57,6 @@ Example usage: env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } - err := exec.Execv(exec.ExecvOptions{ Args: args, Env: env, From 7d46902a1b0b74ea0929a2a6d2963991f536c614 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:34:54 +0200 Subject: [PATCH 60/90] - --- cmd/bundle/exec.go | 76 ---------------------------------------------- cmd/bundle/run.go | 19 ++++++++++-- 2 files changed, 17 insertions(+), 78 deletions(-) delete mode 100644 cmd/bundle/exec.go diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go deleted file mode 100644 index fc950157631..00000000000 --- a/cmd/bundle/exec.go +++ /dev/null @@ -1,76 +0,0 @@ -package bundle - -import ( - "fmt" - "strings" - - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/auth" - "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/exec" - "github.com/spf13/cobra" -) - -func newExecCommand() *cobra.Command { - execCmd := &cobra.Command{ - Use: "exec", - Short: "Execute a command using the same authentication context as the bundle", - Args: cobra.MinimumNArgs(1), - Long: `Execute a command using the same authentication context as the bundle - -Note: The current working directory of the provided command will be set to the root -of the bundle. - -Authentication to the input command will be provided by setting the appropriate -environment variables that Databricks tools use to authenticate. - -Example usage: -1. databricks bundle exec -- echo "hello, world" -2. databricks bundle exec -- /bin/bash -c "echo hello" -3. databricks bundle exec -- uv run pytest`, - RunE: func(cmd *cobra.Command, args []string) error { - if cmd.ArgsLenAtDash() != 0 { - return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) - } - - // Load the bundle configuration to get the authentication credentials. - b, diags := root.MustConfigureBundle(cmd) - if diags.HasError() { - return diags.Error() - } - - env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) - - // If user has specified a target, pass it to the child command. - // - // This is only useful for when the Databricks CLI is the child command. - if target := root.GetTarget(cmd); target != "" { - env = append(env, "DATABRICKS_CONFIG_TARGET="+target) - } - - // If the bundle has a profile configured, explicitly pass it to the child command. - // - // This is only useful for when the Databricks CLI is the child command, - // since if we do not explicitly pass the profile, the CLI will use the - // auth configured in the bundle YAML configuration (if any). - if b.Config.Workspace.Profile != "" { - env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) - } - - err := exec.Execv(exec.ExecvOptions{ - Args: args, - Env: env, - Dir: b.BundleRootPath, - Stderr: cmd.ErrOrStderr(), - Stdout: cmd.OutOrStdout(), - }) - if err != nil { - return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) - } - - return nil - }, - } - - return execCmd -} diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 15c01c18c8f..52c81affc8a 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -76,7 +76,7 @@ func keyToRunner(b *bundle.Bundle, arg string) (run.Runner, error) { func newRunCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "run [flags] KEY", + Use: "run [flags] [KEY]", Short: "Run a job or pipeline update", Long: `Run the job or pipeline identified by KEY. @@ -96,6 +96,22 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. + +--------------------------------------------------------- + +You can also use the bundle run command to execute scripts / commands in the same +authentication context as the bundle. + +Note: The current working directory of the provided command will be set to the root +of the bundle. + +Authentication to the input command will be provided by setting the appropriate +environment variables that Databricks tools use to authenticate. + +Example usage: +1. databricks bundle exec -- echo "hello, world" +2. databricks bundle exec -- /bin/bash -c "echo hello" +3. databricks bundle exec -- uv run pytest `, } @@ -220,7 +236,6 @@ task or a Python wheel task, the second example applies. return cmd } -// TODO: Test singal propogation in the acceptance testS? func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) From c0c1ac65c8b44e47ba31666e3f720f40c45b2404 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:37:20 +0200 Subject: [PATCH 61/90] - --- cmd/bundle/bundle.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index e0818c2f977..fb88cd7d05a 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -28,6 +28,5 @@ func New() *cobra.Command { cmd.AddCommand(newDebugCommand()) cmd.AddCommand(deployment.NewDeploymentCommand()) cmd.AddCommand(newOpenCommand()) - cmd.AddCommand(newExecCommand()) return cmd } From 3b579252683c1308fb37b7273a0c62b6c2c09f4c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:37:32 +0200 Subject: [PATCH 62/90] - --- libs/exec/execv_windows.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index a2738c67ca3..ac9ce786db0 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -1,4 +1,5 @@ //go:build windows + package exec import ( From e5e6ea4ea90b1da88e441b9dd0a1df81a6a1b944 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:38:55 +0200 Subject: [PATCH 63/90] cleanup --- cmd/bundle/run.go | 9 +++++++-- libs/exec/execv_windows.go | 7 ------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 52c81affc8a..d8263068fd2 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -256,8 +256,13 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { } return exec.Execv(exec.ExecvOptions{ - Args: args, - Env: env, + Args: args, + Env: env, + // Execute all scripts from the bundle root directory. This behavior can + // be surprising in isolation, but we do it to keep the behavior consistent + // for both these cases: + // 1. One shot commands like `databricks bundle exec -- echo hello` + // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. Dir: b.BundleRootPath, Stderr: cmd.ErrOrStderr(), Stdout: cmd.OutOrStdout(), diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index ac9ce786db0..1fd525a8e4f 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -21,15 +21,8 @@ func execv(opts ExecvOptions) error { return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) } - // TODO: Validate atleast one arg before calling execv. cmd := exec.Command(path, opts.Args[1:]...) - // TODO: Move this comment. - // Execute all scripts from the bundle root directory. This behavior can - // be surprising in isolation, but we do it to keep the behavior consistent - // for both these cases: - // 1. One shot commands like `databricks bundle exec -- echo hello` - // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. cmd.Dir = opts.Dir cmd.Env = opts.Env From b0bb768d4440408643b7bb2c94be42d36a9af49e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:41:36 +0200 Subject: [PATCH 64/90] - --- acceptance/bundle/help/bundle-run/output.txt | 18 +++++++++++++++++- acceptance/bundle/help/bundle/output.txt | 1 - .../bundle/run/inline-script/basic/output.txt | 2 +- .../bundle/run/inline-script/basic/script | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 4b9efbf2ae2..d86ad53740f 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -19,8 +19,24 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. +--------------------------------------------------------- + +You can also use the bundle run command to execute scripts / commands in the same +authentication context as the bundle. + +Note: The current working directory of the provided command will be set to the root +of the bundle. + +Authentication to the input command will be provided by setting the appropriate +environment variables that Databricks tools use to authenticate. + +Example usage: +1. databricks bundle exec -- echo "hello, world" +2. databricks bundle exec -- /bin/bash -c "echo hello" +3. databricks bundle exec -- uv run pytest + Usage: - databricks bundle run [flags] KEY + databricks bundle run [flags] [KEY] Job Flags: --params stringToString comma separated k=v pairs for job parameters (default []) diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index bb885c80e53..fc6dd623dde 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -11,7 +11,6 @@ Available Commands: deploy Deploy bundle deployment Deployment related commands destroy Destroy deployed bundle resources - exec Execute a command using the same authentication context as the bundle generate Generate bundle configuration init Initialize using a bundle template open Open a resource in the browser diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index 70ffeb1f0c6..294980e3c59 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -13,5 +13,5 @@ Exit code: 5 >>> cat stderr.txt ->>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) +>>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index 350705035c4..c1b5e70ec0a 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -8,6 +8,6 @@ errcode trace $CLI bundle run -- --help errcode trace $CLI bundle run -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt +trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt trace cat stderr.txt rm stderr.txt From fa6151bdb24737233f0c7ca825b9d5ffa354e5cb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:44:30 +0200 Subject: [PATCH 65/90] - --- acceptance/bundle/help/bundle-run/output.txt | 8 +++++--- cmd/bundle/run.go | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index d86ad53740f..4f0a6b107c7 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -31,9 +31,11 @@ Authentication to the input command will be provided by setting the appropriate environment variables that Databricks tools use to authenticate. Example usage: -1. databricks bundle exec -- echo "hello, world" -2. databricks bundle exec -- /bin/bash -c "echo hello" -3. databricks bundle exec -- uv run pytest +1. databricks bundle run -- echo "hello, world" +2. databricks bundle run -- /bin/bash -c "echo hello" +3. databricks bundle run -- uv run pytest + +--------------------------------------------------------- Usage: databricks bundle run [flags] [KEY] diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index d8263068fd2..69e7a0f5832 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -109,9 +109,11 @@ Authentication to the input command will be provided by setting the appropriate environment variables that Databricks tools use to authenticate. Example usage: -1. databricks bundle exec -- echo "hello, world" -2. databricks bundle exec -- /bin/bash -c "echo hello" -3. databricks bundle exec -- uv run pytest +1. databricks bundle run -- echo "hello, world" +2. databricks bundle run -- /bin/bash -c "echo hello" +3. databricks bundle run -- uv run pytest + +--------------------------------------------------------- `, } From a32d8b1ab8d075b46238c6ea1c8d2a43d915690a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 19:10:41 +0200 Subject: [PATCH 66/90] -" " ; --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 1c129d08cf2..0d69cd7c46f 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -3,6 +3,7 @@ ## Release v0.248.0 ### Notable Changes +* Added inline script execution support to bundle run. You can now run scripts in the same authentication context as a DAB using the databricks bundle run command. ([#2413](https://github.com/databricks/cli/pull/2413)) ### Dependency updates From a098d4a9ff7db6e8905fde423c5d42a4073859c7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 15 Apr 2025 11:25:56 +0200 Subject: [PATCH 67/90] - --- cmd/bundle/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 69e7a0f5832..33023117267 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -263,7 +263,7 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { // Execute all scripts from the bundle root directory. This behavior can // be surprising in isolation, but we do it to keep the behavior consistent // for both these cases: - // 1. One shot commands like `databricks bundle exec -- echo hello` + // 1. One shot commands like `databricks bundle run -- echo hello` // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. Dir: b.BundleRootPath, Stderr: cmd.ErrOrStderr(), From fcd584b0f8b14b1433c9e68c4c10a35be633ce26 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Apr 2025 19:48:56 +0200 Subject: [PATCH 68/90] newline test --- acceptance/bundle/run/inline-script/basic/output.txt | 3 +++ acceptance/bundle/run/inline-script/basic/script | 3 +++ 2 files changed, 6 insertions(+) diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index 294980e3c59..61cb70abca1 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -15,3 +15,6 @@ Exit code: 5 >>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) Hello + +>>> [CLI] bundle run -- printf hello +hello \ No newline at end of file diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index c1b5e70ec0a..d9ca06d7f05 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -11,3 +11,6 @@ errcode trace $CLI bundle run -- bash -c "exit 5" trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt trace cat stderr.txt rm stderr.txt + +# no newline +trace $CLI bundle run -- printf "hello" From f1f921da1057b21f7ef19facefa0e8d96c5953a0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 17 Apr 2025 14:09:03 +0200 Subject: [PATCH 69/90] - --- acceptance/bundle/run/inline-script/basic/output.txt | 3 +++ acceptance/bundle/run/inline-script/basic/script | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index 61cb70abca1..7d6a28f6aff 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -16,5 +16,8 @@ Exit code: 5 >>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) Hello +>>> cat - +abc + >>> [CLI] bundle run -- printf hello hello \ No newline at end of file diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index d9ca06d7f05..a19486dd986 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -12,5 +12,10 @@ trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr) trace cat stderr.txt rm stderr.txt +# stdin should be passed through +echo "abc" > abc.txt +trace cat - < abc.txt +rm abc.txt + # no newline trace $CLI bundle run -- printf "hello" From 5965b9054ac7d260b3e92e191cded3d2dd1243f4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 19:27:52 +0200 Subject: [PATCH 70/90] return via chunks --- libs/exec/execv_windows.go | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 1fd525a8e4f..373dd17658c 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -3,8 +3,8 @@ package exec import ( - "bufio" "fmt" + "io" "os" "os/exec" "strings" @@ -48,29 +48,13 @@ func execv(opts ExecvOptions) error { var stdoutErr error go func() { defer wg.Done() - - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - _, err = fmt.Fprintln(opts.Stdout, scanner.Text()) - if err != nil { - stdoutErr = err - break - } - } + _, stdoutErr = io.Copy(opts.Stdout, stdout) }() var stderrErr error go func() { defer wg.Done() - - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - _, err = fmt.Fprintln(opts.Stderr, scanner.Text()) - if err != nil { - stderrErr = err - break - } - } + _, stderrErr = io.Copy(opts.Stderr, stderr) }() wg.Wait() From 19cf6d3cf9fd8327f59ab9e150507485e18dea4b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 19:33:20 +0200 Subject: [PATCH 71/90] add stdin --- cmd/bundle/run.go | 1 + libs/exec/execv.go | 5 +++-- libs/exec/execv_windows.go | 20 +++++++++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 33023117267..b2568ad853e 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -268,5 +268,6 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { Dir: b.BundleRootPath, Stderr: cmd.ErrOrStderr(), Stdout: cmd.OutOrStdout(), + Stdin: cmd.InOrStdin(), }) } diff --git a/libs/exec/execv.go b/libs/exec/execv.go index c477d76dd40..cd97779bbfb 100644 --- a/libs/exec/execv.go +++ b/libs/exec/execv.go @@ -10,9 +10,10 @@ type ExecvOptions struct { Env []string Dir string - // Stderr and Stdout are only used for Windows where execv is not supported. - // For Unix, the Stdout and Stderr are automatically inherited during the exec + // Stdin, Stderr and Stdout are only used for Windows where execv is not supported. + // For Unix, these are automatically inherited during the exec // system call. + Stdin io.Reader Stderr io.Writer Stdout io.Writer } diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 373dd17658c..9e36f65e217 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -26,6 +26,12 @@ func execv(opts ExecvOptions) error { cmd.Dir = opts.Dir cmd.Env = opts.Env + // Setup Stdin pipe + stdin, err := cmd.StdinPipe() + if err != nil { + return fmt.Errorf("creating stdin pipe failed: %w", err) + } + stdout, err := cmd.StdoutPipe() if err != nil { return fmt.Errorf("creating stdout pipe failed: %w", err) @@ -43,7 +49,15 @@ func execv(opts ExecvOptions) error { } var wg sync.WaitGroup - wg.Add(2) + wg.Add(3) // Increment WaitGroup for stdin goroutine + + // Goroutine to copy stdin + var stdinErr error + go func() { + defer wg.Done() + defer stdin.Close() // Close stdin pipe when copying is done + _, stdinErr = io.Copy(stdin, opts.Stdin) + }() var stdoutErr error go func() { @@ -67,6 +81,10 @@ func execv(opts ExecvOptions) error { return fmt.Errorf("writing stderr failed: %w", stderrErr) } + if stdinErr != nil { + return fmt.Errorf("reading from stdin failed: %w", stdinErr) + } + err = cmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { os.Exit(exitErr.ExitCode()) From 37c5fd4faa6b3c71e4886e565162369f164da7c3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 19:36:07 +0200 Subject: [PATCH 72/90] comments --- libs/exec/execv.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libs/exec/execv.go b/libs/exec/execv.go index cd97779bbfb..53e87be0eda 100644 --- a/libs/exec/execv.go +++ b/libs/exec/execv.go @@ -6,16 +6,22 @@ import "io" // that users can read to figure out the original CWD. I'll do that when // adding support for the scripts section. type ExecvOptions struct { + // Args is the name of the command to run and its arguments. + // Eg: ["echo", "hello"] for "echo hello" Args []string - Env []string - Dir string + + // Env is set the environment variables to set in the child process. + Env []string + + // Dir is the working directory of the child process. + Dir string // Stdin, Stderr and Stdout are only used for Windows where execv is not supported. // For Unix, these are automatically inherited during the exec // system call. Stdin io.Reader - Stderr io.Writer Stdout io.Writer + Stderr io.Writer } func Execv(opts ExecvOptions) error { From 772d3f8a597422df8c4aa908174ff21f1ae15d63 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 19:37:00 +0200 Subject: [PATCH 73/90] - --- libs/exec/execv_unix.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/exec/execv_unix.go b/libs/exec/execv_unix.go index bbb896ad037..e81e51d07d0 100644 --- a/libs/exec/execv_unix.go +++ b/libs/exec/execv_unix.go @@ -15,7 +15,8 @@ func execv(opts ExecvOptions) error { return fmt.Errorf("changing directory to %s failed: %w", opts.Dir, err) } - // execve syscall does not perform PATH lookup and + // execve syscall does not perform PATH lookup. Thus we need to query path + // before making the exec syscall. path, err := exec.LookPath(opts.Args[0]) if err != nil { return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) From 8235a15122132c820e28e3f8cef518e3b5a45b08 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 19:49:51 +0200 Subject: [PATCH 74/90] - --- libs/exec/execv_windows.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 9e36f65e217..cffdb37bd37 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -12,8 +12,7 @@ import ( ) // Note: Windows does not support an execv syscall that replaces the current process. -// Thus we emulate that by launching a child process and streaming the output and returning -// the exit code. +// To emulate this, we create a child process, stream the input and output, and return the exit code. // ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/execv-wexecv?view=msvc-170 func execv(opts ExecvOptions) error { path, err := exec.LookPath(opts.Args[0]) From 33ff09d3b41874bc2978dbb999ec710eb6c0db92 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 20:31:09 +0200 Subject: [PATCH 75/90] - --- .../target-is-passed/default/out.requests.txt | 30 +++++++++++++++++++ .../target-is-passed/default/output.txt | 7 ++--- .../target-is-passed/default/script | 2 +- .../from_flag/out.requests.txt | 30 +++++++++++++++++++ .../target-is-passed/from_flag/output.txt | 7 ++--- .../target-is-passed/from_flag/script | 2 +- cmd/bundle/run.go | 2 +- 7 files changed, 67 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt index c4a1034826d..57a637ce0a5 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt @@ -7,3 +7,33 @@ "method": "GET", "path": "/api/2.0/preview/scim/v2/Me" } +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/foobar/pat/files" + } +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt index 704a6ac915e..513329a539b 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt @@ -1,6 +1,3 @@ ->>> [CLI] bundle run -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} +>>> [CLI] bundle run -- [CLI] bundle validate -o json +"pat" diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script index f9d3610b4e0..16b75725145 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # Should use default target, which is pat based authentication -trace $CLI bundle run -- $CLI current-user me +trace $CLI bundle run -- $CLI bundle validate -o json| jq .bundle.target diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt index bbad66cf8bb..f4fb08cab2c 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt @@ -25,3 +25,33 @@ "method": "GET", "path": "/api/2.0/preview/scim/v2/Me" } +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/foobar/oauth/files" + } +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt index 9d568529fda..fb8a3292890 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt @@ -1,6 +1,3 @@ ->>> [CLI] bundle run -t oauth -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} +>>> [CLI] bundle run -t oauth -- [CLI] bundle validate -o json +"oauth" diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script index fcae775022c..62c438ff012 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # Explicitly select oauth target -trace $CLI bundle run -t oauth -- $CLI current-user me +trace $CLI bundle run -t oauth -- $CLI bundle validate -o json| jq .bundle.target diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index b2568ad853e..563a586a2ee 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -245,7 +245,7 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { // // This is only useful for when the Databricks CLI is the child command. if target := root.GetTarget(cmd); target != "" { - env = append(env, "DATABRICKS_CONFIG_TARGET="+target) + env = append(env, "DATABRICKS_BUNDLE_TARGET="+target) } // If the bundle has a profile configured, explicitly pass it to the child command. From 94ab0376ce9c88ed22b2dcf243cad1190a60a948 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 20:36:00 +0200 Subject: [PATCH 76/90] - --- cmd/bundle/run.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 563a586a2ee..86e9ad854b3 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run" @@ -239,13 +240,13 @@ Example usage: } func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { - env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) + cmdEnv := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. // // This is only useful for when the Databricks CLI is the child command. if target := root.GetTarget(cmd); target != "" { - env = append(env, "DATABRICKS_BUNDLE_TARGET="+target) + cmdEnv = append(cmdEnv, env.TargetVariable+"="+target) } // If the bundle has a profile configured, explicitly pass it to the child command. @@ -254,12 +255,12 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { // since if we do not explicitly pass the profile, the CLI will use the // auth configured in the bundle YAML configuration (if any). if b.Config.Workspace.Profile != "" { - env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) + cmdEnv = append(cmdEnv, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } return exec.Execv(exec.ExecvOptions{ Args: args, - Env: env, + Env: cmdEnv, // Execute all scripts from the bundle root directory. This behavior can // be surprising in isolation, but we do it to keep the behavior consistent // for both these cases: From 1b8e7df4c0d1178ddd19a05b3eba21d5fbcb71cd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 20:38:35 +0200 Subject: [PATCH 77/90] - --- .wsignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.wsignore b/.wsignore index 827ebf4332c..389e8aa2d59 100644 --- a/.wsignore +++ b/.wsignore @@ -14,6 +14,9 @@ experimental/python/docs/images/databricks-logo.svg # "bundle run" has trailing whitespace: acceptance/bundle/integration_whl/*/output.txt +# "bundle run" inline script execution has trailing whitespace: +acceptance/bundle/run/inline-script/basic/output.txt + # "bundle init" has trailing whitespace: acceptance/bundle/templates-machinery/helpers-error/output.txt From 8fe87581085a7b8aa18aff987e0e1040d074c321 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 20:40:47 +0200 Subject: [PATCH 78/90] - --- cmd/bundle/run.go | 4 ++-- cmd/root/bundle.go | 6 +++--- cmd/root/bundle_test.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 86e9ad854b3..263c8307cc6 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -245,8 +245,8 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { // If user has specified a target, pass it to the child command. // // This is only useful for when the Databricks CLI is the child command. - if target := root.GetTarget(cmd); target != "" { - cmdEnv = append(cmdEnv, env.TargetVariable+"="+target) + if b.Config.Bundle.Target != "" { + cmdEnv = append(cmdEnv, env.TargetVariable+"="+b.Config.Bundle.Target) } // If the bundle has a profile configured, explicitly pass it to the child command. diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go index a5fc24021a0..99c278e2f5c 100644 --- a/cmd/root/bundle.go +++ b/cmd/root/bundle.go @@ -13,8 +13,8 @@ import ( "golang.org/x/exp/maps" ) -// GetTarget returns the name of the target to operate in. -func GetTarget(cmd *cobra.Command) (value string) { +// getTarget returns the name of the target to operate in. +func getTarget(cmd *cobra.Command) (value string) { target, isFlagSet := targetFlagValue(cmd) if isFlagSet { return target @@ -78,7 +78,7 @@ func configureBundle(cmd *cobra.Command, b *bundle.Bundle) (*bundle.Bundle, diag // Load bundle and select target. ctx := cmd.Context() var diags diag.Diagnostics - if target := GetTarget(cmd); target == "" { + if target := getTarget(cmd); target == "" { diags = phases.LoadDefaultTarget(ctx, b) } else { diags = phases.LoadNamedTarget(ctx, b, target) diff --git a/cmd/root/bundle_test.go b/cmd/root/bundle_test.go index 3765b3e60bc..45a476ff3f4 100644 --- a/cmd/root/bundle_test.go +++ b/cmd/root/bundle_test.go @@ -216,7 +216,7 @@ func TestTargetFlagFull(t *testing.T) { err := Execute(ctx, cmd) assert.NoError(t, err) - assert.Equal(t, "development", GetTarget(cmd)) + assert.Equal(t, "development", getTarget(cmd)) } func TestTargetFlagShort(t *testing.T) { @@ -228,7 +228,7 @@ func TestTargetFlagShort(t *testing.T) { err := Execute(ctx, cmd) assert.NoError(t, err) - assert.Equal(t, "production", GetTarget(cmd)) + assert.Equal(t, "production", getTarget(cmd)) } // TODO: remove when environment flag is fully deprecated @@ -242,5 +242,5 @@ func TestTargetEnvironmentFlag(t *testing.T) { err := Execute(ctx, cmd) assert.NoError(t, err) - assert.Equal(t, "development", GetTarget(cmd)) + assert.Equal(t, "development", getTarget(cmd)) } From 913af8a97cd3561c2b98f42eec32ac6d753be903 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 20:52:23 +0200 Subject: [PATCH 79/90] - --- .../bundle/run/inline-script/cwd/output.txt | 2 +- cmd/bundle/run.go | 17 +++++++++-------- libs/exec/execv.go | 3 --- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/run/inline-script/cwd/output.txt b/acceptance/bundle/run/inline-script/cwd/output.txt index 8a571d5de87..daa86d33738 100644 --- a/acceptance/bundle/run/inline-script/cwd/output.txt +++ b/acceptance/bundle/run/inline-script/cwd/output.txt @@ -5,4 +5,4 @@ [TEST_TMP_DIR]/a/b/c >>> [CLI] bundle run -- python3 -c import os; print(os.getcwd()) -[TEST_TMP_DIR] +[TEST_TMP_DIR]/a/b/c diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 263c8307cc6..705e7558a7a 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/terraform" @@ -258,15 +259,15 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { cmdEnv = append(cmdEnv, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } + dir, err := os.Getwd() + if err != nil { + return err + } + return exec.Execv(exec.ExecvOptions{ - Args: args, - Env: cmdEnv, - // Execute all scripts from the bundle root directory. This behavior can - // be surprising in isolation, but we do it to keep the behavior consistent - // for both these cases: - // 1. One shot commands like `databricks bundle run -- echo hello` - // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. - Dir: b.BundleRootPath, + Args: args, + Env: cmdEnv, + Dir: dir, Stderr: cmd.ErrOrStderr(), Stdout: cmd.OutOrStdout(), Stdin: cmd.InOrStdin(), diff --git a/libs/exec/execv.go b/libs/exec/execv.go index 53e87be0eda..16c5b17f44a 100644 --- a/libs/exec/execv.go +++ b/libs/exec/execv.go @@ -2,9 +2,6 @@ package exec import "io" -// TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable -// that users can read to figure out the original CWD. I'll do that when -// adding support for the scripts section. type ExecvOptions struct { // Args is the name of the command to run and its arguments. // Eg: ["echo", "hello"] for "echo hello" From 9209890530d64a1a823f0ae775a00b7a2cdf124f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 20:56:49 +0200 Subject: [PATCH 80/90] - --- acceptance/bundle/help/bundle-run/output.txt | 3 --- cmd/bundle/run.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 4f0a6b107c7..c457edc2796 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -24,9 +24,6 @@ task or a Python wheel task, the second example applies. You can also use the bundle run command to execute scripts / commands in the same authentication context as the bundle. -Note: The current working directory of the provided command will be set to the root -of the bundle. - Authentication to the input command will be provided by setting the appropriate environment variables that Databricks tools use to authenticate. diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 705e7558a7a..65ba054294a 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -104,9 +104,6 @@ task or a Python wheel task, the second example applies. You can also use the bundle run command to execute scripts / commands in the same authentication context as the bundle. -Note: The current working directory of the provided command will be set to the root -of the bundle. - Authentication to the input command will be provided by setting the appropriate environment variables that Databricks tools use to authenticate. From 7ba556e396c672808533e9af1c8a2f13eadb99bd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 21:05:33 +0200 Subject: [PATCH 81/90] clearup --- libs/exec/execv_windows.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index cffdb37bd37..cbb6e05100e 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -25,7 +25,6 @@ func execv(opts ExecvOptions) error { cmd.Dir = opts.Dir cmd.Env = opts.Env - // Setup Stdin pipe stdin, err := cmd.StdinPipe() if err != nil { return fmt.Errorf("creating stdin pipe failed: %w", err) @@ -54,7 +53,10 @@ func execv(opts ExecvOptions) error { var stdinErr error go func() { defer wg.Done() - defer stdin.Close() // Close stdin pipe when copying is done + // Close stdin pipe when we are done copying + // from the parent process. + defer stdin.Close() + _, stdinErr = io.Copy(stdin, opts.Stdin) }() @@ -92,5 +94,8 @@ func execv(opts ExecvOptions) error { return fmt.Errorf("running %s failed: %w", strings.Join(opts.Args, " "), err) } - return nil + // Unix implementation of execv never returns control to the CLI process. + // To emulate this behavior, we exit early here if the child process exits + // successfully. + os.Exit(0) } From 45b7f5d7a8ce399bb6235fc114240531ebdb1fd1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Apr 2025 23:03:01 +0200 Subject: [PATCH 82/90] fix build --- libs/exec/execv_windows.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index cbb6e05100e..0e40c9133ad 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -98,4 +98,5 @@ func execv(opts ExecvOptions) error { // To emulate this behavior, we exit early here if the child process exits // successfully. os.Exit(0) + return nil } From 3a1b515890b4a5fe5337c07bf22b606e508a1b31 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 22 Apr 2025 23:37:53 +0200 Subject: [PATCH 83/90] - --- libs/exec/execv_windows.go | 61 +++----------------------------------- 1 file changed, 4 insertions(+), 57 deletions(-) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 0e40c9133ad..4626ca1fbe0 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -4,11 +4,9 @@ package exec import ( "fmt" - "io" "os" "os/exec" "strings" - "sync" ) // Note: Windows does not support an execv syscall that replaces the current process. @@ -22,70 +20,19 @@ func execv(opts ExecvOptions) error { cmd := exec.Command(path, opts.Args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Dir = opts.Dir cmd.Env = opts.Env - stdin, err := cmd.StdinPipe() - if err != nil { - return fmt.Errorf("creating stdin pipe failed: %w", err) - } - - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("creating stdout pipe failed: %w", err) - } - - stderr, err := cmd.StderrPipe() - if err != nil { - return fmt.Errorf("creating stderr pipe failed: %w", err) - } - // Start the child command. err = cmd.Start() if err != nil { return fmt.Errorf(" %s failed: %w", strings.Join(opts.Args, " "), err) } - var wg sync.WaitGroup - wg.Add(3) // Increment WaitGroup for stdin goroutine - - // Goroutine to copy stdin - var stdinErr error - go func() { - defer wg.Done() - // Close stdin pipe when we are done copying - // from the parent process. - defer stdin.Close() - - _, stdinErr = io.Copy(stdin, opts.Stdin) - }() - - var stdoutErr error - go func() { - defer wg.Done() - _, stdoutErr = io.Copy(opts.Stdout, stdout) - }() - - var stderrErr error - go func() { - defer wg.Done() - _, stderrErr = io.Copy(opts.Stderr, stderr) - }() - - wg.Wait() - - if stdoutErr != nil { - return fmt.Errorf("writing stdout failed: %w", stdoutErr) - } - - if stderrErr != nil { - return fmt.Errorf("writing stderr failed: %w", stderrErr) - } - - if stdinErr != nil { - return fmt.Errorf("reading from stdin failed: %w", stdinErr) - } - err = cmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { os.Exit(exitErr.ExitCode()) From d951f58f38e4994b1fb35ac345cd7259140b0a30 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 23 Apr 2025 11:05:27 +0200 Subject: [PATCH 84/90] - --- .wsignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.wsignore b/.wsignore index 389e8aa2d59..260b8cc0d36 100644 --- a/.wsignore +++ b/.wsignore @@ -14,7 +14,8 @@ experimental/python/docs/images/databricks-logo.svg # "bundle run" has trailing whitespace: acceptance/bundle/integration_whl/*/output.txt -# "bundle run" inline script execution has trailing whitespace: +# This file does not end with a trailing newline, intentionally because this +# tests whether output is streamed to stdout without newlines acting as a delimiter. acceptance/bundle/run/inline-script/basic/output.txt # "bundle init" has trailing whitespace: From 5676fa926262673c572e2141cf72e35551c8a435 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 23 Apr 2025 15:53:35 +0200 Subject: [PATCH 85/90] - --- cmd/bundle/run.go | 3 --- libs/exec/execv.go | 9 --------- 2 files changed, 12 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 65ba054294a..11da9fd52a7 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -265,8 +265,5 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { Args: args, Env: cmdEnv, Dir: dir, - Stderr: cmd.ErrOrStderr(), - Stdout: cmd.OutOrStdout(), - Stdin: cmd.InOrStdin(), }) } diff --git a/libs/exec/execv.go b/libs/exec/execv.go index 16c5b17f44a..542de00574e 100644 --- a/libs/exec/execv.go +++ b/libs/exec/execv.go @@ -1,7 +1,5 @@ package exec -import "io" - type ExecvOptions struct { // Args is the name of the command to run and its arguments. // Eg: ["echo", "hello"] for "echo hello" @@ -12,13 +10,6 @@ type ExecvOptions struct { // Dir is the working directory of the child process. Dir string - - // Stdin, Stderr and Stdout are only used for Windows where execv is not supported. - // For Unix, these are automatically inherited during the exec - // system call. - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer } func Execv(opts ExecvOptions) error { From 93d7ae9e746d785461215318b009fb8ffae66bbe Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 23 Apr 2025 15:57:41 +0200 Subject: [PATCH 86/90] - --- .wsignore | 3 --- acceptance/bundle/run/inline-script/basic/output.txt | 2 +- acceptance/bundle/run/inline-script/basic/script | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.wsignore b/.wsignore index 260b8cc0d36..a669da94ff6 100644 --- a/.wsignore +++ b/.wsignore @@ -14,9 +14,6 @@ experimental/python/docs/images/databricks-logo.svg # "bundle run" has trailing whitespace: acceptance/bundle/integration_whl/*/output.txt -# This file does not end with a trailing newline, intentionally because this -# tests whether output is streamed to stdout without newlines acting as a delimiter. -acceptance/bundle/run/inline-script/basic/output.txt # "bundle init" has trailing whitespace: acceptance/bundle/templates-machinery/helpers-error/output.txt diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index 7d6a28f6aff..070a954fd52 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -20,4 +20,4 @@ Hello abc >>> [CLI] bundle run -- printf hello -hello \ No newline at end of file +hello diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index a19486dd986..c00cb3e701e 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -19,3 +19,6 @@ rm abc.txt # no newline trace $CLI bundle run -- printf "hello" + +# print newline to comply with lint +printf "\n" From cc552a8125a35fe997d6c634322b342f21fcb09b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 23 Apr 2025 16:04:36 +0200 Subject: [PATCH 87/90] - --- .../profile-is-passed/from_bundle/.databrickscfg | 3 --- .../profile-is-passed/from_bundle/databricks.yml | 5 ----- .../profile-is-passed/from_bundle/out.requests.txt | 9 --------- .../profile-is-passed/from_bundle/output.txt | 6 ------ .../profile-is-passed/from_bundle/script | 10 ---------- cmd/bundle/run.go | 6 +++--- 6 files changed, 3 insertions(+), 36 deletions(-) delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg deleted file mode 100644 index 405e344a02e..00000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg +++ /dev/null @@ -1,3 +0,0 @@ -[someprofile] -host = $DATABRICKS_HOST -token = $DATABRICKS_TOKEN diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml deleted file mode 100644 index 5de7d1d9680..00000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml +++ /dev/null @@ -1,5 +0,0 @@ -bundle: - name: foobar - -workspace: - profile: someprofile diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt deleted file mode 100644 index c4a1034826d..00000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt +++ /dev/null @@ -1,9 +0,0 @@ -{ - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt deleted file mode 100644 index 704a6ac915e..00000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt +++ /dev/null @@ -1,6 +0,0 @@ - ->>> [CLI] bundle run -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script deleted file mode 100644 index 28bc0f1b84e..00000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script +++ /dev/null @@ -1,10 +0,0 @@ -# Replace placeholder with an actual host URL and token -envsubst < .databrickscfg > out && mv out .databrickscfg -export DATABRICKS_CONFIG_FILE=.databrickscfg - -# Credentials will be picked up from .databrickscfg. Unset existing credentials. -unset DATABRICKS_HOST -unset DATABRICKS_TOKEN - -# This should use the profile configured in the bundle, i.e. PAT auth -trace $CLI bundle run -- $CLI current-user me diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 11da9fd52a7..3713f0b80b6 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -262,8 +262,8 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { } return exec.Execv(exec.ExecvOptions{ - Args: args, - Env: cmdEnv, - Dir: dir, + Args: args, + Env: cmdEnv, + Dir: dir, }) } From 573335075f38bf495cd52c76d219930d35859c56 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 24 Apr 2025 12:43:10 +0200 Subject: [PATCH 88/90] - --- .wsignore | 1 - acceptance/bundle/run/inline-script/basic/script | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.wsignore b/.wsignore index a669da94ff6..827ebf4332c 100644 --- a/.wsignore +++ b/.wsignore @@ -14,7 +14,6 @@ experimental/python/docs/images/databricks-logo.svg # "bundle run" has trailing whitespace: acceptance/bundle/integration_whl/*/output.txt - # "bundle init" has trailing whitespace: acceptance/bundle/templates-machinery/helpers-error/output.txt diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index c00cb3e701e..a28bdb9aa61 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -20,5 +20,5 @@ rm abc.txt # no newline trace $CLI bundle run -- printf "hello" -# print newline to comply with lint +# print newline to comply with whitespace linter printf "\n" From 63eeed10b8ea657984ac4fe1c98112504f48c331 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 24 Apr 2025 12:44:38 +0200 Subject: [PATCH 89/90] - --- libs/exec/execv_windows.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 4626ca1fbe0..ded74d7f93b 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -10,7 +10,8 @@ import ( ) // Note: Windows does not support an execv syscall that replaces the current process. -// To emulate this, we create a child process, stream the input and output, and return the exit code. +// To emulate this, we create a child process, pass the stdin, stdout and stderr file descriptors, +// and return the exit code. // ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/execv-wexecv?view=msvc-170 func execv(opts ExecvOptions) error { path, err := exec.LookPath(opts.Args[0]) From 45da83fa6443fe3a040ae0ab6af363ae481ccc1a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 24 Apr 2025 15:14:50 +0200 Subject: [PATCH 90/90] address comments --- acceptance/bundle/run/inline-script/basic/output.txt | 2 -- acceptance/bundle/run/inline-script/basic/script | 2 +- libs/exec/execv_windows.go | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index 070a954fd52..385f7759669 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -12,8 +12,6 @@ Exit code: 1 Exit code: 5 >>> cat stderr.txt - ->>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) Hello >>> cat - diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index a28bdb9aa61..714e6074793 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -8,7 +8,7 @@ errcode trace $CLI bundle run -- --help errcode trace $CLI bundle run -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt +$CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt trace cat stderr.txt rm stderr.txt diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index ded74d7f93b..4bb73973d1d 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -28,7 +28,6 @@ func execv(opts ExecvOptions) error { cmd.Dir = opts.Dir cmd.Env = opts.Env - // Start the child command. err = cmd.Start() if err != nil { return fmt.Errorf(" %s failed: %w", strings.Join(opts.Args, " "), err)