diff --git a/CHANGELOG.md b/CHANGELOG.md index 00f08f58..0bdde4c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- Add opt-in command substitution via `$(command)` syntax, enabled with `execute_commands=True` on `load_dotenv()` and `dotenv_values()`, or `--execute-commands` on the CLI + ### Fixed - An unquoted empty value followed by an inline comment (e.g. `KEY= # comment`) is now parsed as an empty string instead of the comment text by [@Noethix55555] in [#663] diff --git a/README.md b/README.md index a08d6141..ff3b4793 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,26 @@ values defined in the following list: - Default value, if provided. - Empty string. +### Command substitution + +python-dotenv can run shell commands and use their output as variable values +using `$(command)` syntax. This is disabled by default; pass +`execute_commands=True` to `load_dotenv()` or `dotenv_values()` to enable it. + +```bash +GITHUB_TOKEN=$(gh auth token) +``` + +Only use command substitution with `.env` files you trust. Commands run with +the permissions of the current process. + +Commands containing `)` inside `$(...)` are not supported (for example, +`$(python -c "print(1)")`). Use helper scripts or commands without nested +parentheses instead. + +The CLI flag `--execute-commands` enables this for `dotenv list`, `dotenv get`, +and `dotenv run`. + ## Related Projects - [environs](https://github.com/sloria/environs) diff --git a/src/dotenv/cli.py b/src/dotenv/cli.py index 79613e28..7cfe39dc 100644 --- a/src/dotenv/cli.py +++ b/src/dotenv/cli.py @@ -57,11 +57,24 @@ def enumerate_env() -> Optional[str]: type=click.BOOL, help="Whether to write the dot file as an executable bash script.", ) +@click.option( + "--execute-commands", + is_flag=True, + default=False, + help="Execute $(command) substitutions in values.", +) @click.version_option(version=__version__) @click.pass_context -def cli(ctx: click.Context, file: Any, quote: Any, export: Any) -> None: +def cli( + ctx: click.Context, file: Any, quote: Any, export: Any, execute_commands: bool +) -> None: """This script is used to set, get or unset values from a .env file.""" - ctx.obj = {"QUOTE": quote, "EXPORT": export, "FILE": file} + ctx.obj = { + "QUOTE": quote, + "EXPORT": export, + "FILE": file, + "EXECUTE_COMMANDS": execute_commands, + } @contextmanager @@ -95,7 +108,9 @@ def list_values(ctx: click.Context, output_format: str) -> None: file = ctx.obj["FILE"] with stream_file(file) as stream: - values = dotenv_values(stream=stream) + values = dotenv_values( + stream=stream, execute_commands=ctx.obj["EXECUTE_COMMANDS"] + ) if output_format == "json": click.echo(json.dumps(values, indent=2, sort_keys=True)) @@ -139,7 +154,9 @@ def get(ctx: click.Context, key: Any) -> None: file = ctx.obj["FILE"] with stream_file(file) as stream: - values = dotenv_values(stream=stream) + values = dotenv_values( + stream=stream, execute_commands=ctx.obj["EXECUTE_COMMANDS"] + ) stored_value = values.get(key) if stored_value: @@ -190,7 +207,9 @@ def run(ctx: click.Context, override: bool, commandline: tuple[str, ...]) -> Non ) dotenv_as_dict = { k: v - for (k, v) in dotenv_values(file).items() + for (k, v) in dotenv_values( + file, execute_commands=ctx.obj["EXECUTE_COMMANDS"] + ).items() if v is not None and (override or k not in os.environ) } diff --git a/src/dotenv/main.py b/src/dotenv/main.py index 3123690a..7e22a3e5 100644 --- a/src/dotenv/main.py +++ b/src/dotenv/main.py @@ -10,7 +10,7 @@ from typing import IO, Dict, Iterable, Iterator, Mapping, Optional, Tuple, Union from .parser import Binding, parse_stream -from .variables import parse_variables +from .variables import parse_variables, resolve_commands # A type alias for a string path to be used for the paths in this file. # These paths may flow to `open()` and `os.replace()`. @@ -48,6 +48,7 @@ def __init__( encoding: Optional[str] = None, interpolate: bool = True, override: bool = True, + execute_commands: bool = False, ) -> None: self.dotenv_path: Optional[StrPath] = dotenv_path self.stream: Optional[IO[str]] = stream @@ -56,6 +57,7 @@ def __init__( self.encoding: Optional[str] = encoding self.interpolate: bool = interpolate self.override: bool = override + self.execute_commands: bool = execute_commands @contextmanager def _get_stream(self) -> Iterator[IO[str]]: @@ -79,9 +81,14 @@ def dict(self) -> Dict[str, Optional[str]]: raw_values = self.parse() - if self.interpolate: + if self.interpolate or self.execute_commands: self._dict = OrderedDict( - resolve_variables(raw_values, override=self.override) + resolve_variables( + raw_values, + override=self.override, + interpolate=self.interpolate, + execute_commands=self.execute_commands, + ) ) else: self._dict = OrderedDict(raw_values) @@ -294,6 +301,8 @@ def unset_key( def resolve_variables( values: Iterable[Tuple[str, Optional[str]]], override: bool, + interpolate: bool = True, + execute_commands: bool = False, ) -> Mapping[str, Optional[str]]: new_values: Dict[str, Optional[str]] = {} @@ -301,7 +310,6 @@ def resolve_variables( if value is None: result = None else: - atoms = parse_variables(value) env: Dict[str, Optional[str]] = {} if override: env.update(os.environ) # type: ignore @@ -309,7 +317,15 @@ def resolve_variables( else: env.update(new_values) env.update(os.environ) # type: ignore - result = "".join(atom.resolve(env) for atom in atoms) + + if interpolate: + atoms = parse_variables(value) + result = "".join(atom.resolve(env) for atom in atoms) + else: + result = value + + if execute_commands: + result = resolve_commands(result, env) new_values[name] = result @@ -392,6 +408,7 @@ def load_dotenv( override: bool = False, interpolate: bool = True, encoding: Optional[str] = "utf-8", + execute_commands: bool = False, ) -> bool: """Parse a .env file and then load all the variables found as environment variables. @@ -404,6 +421,7 @@ def load_dotenv( from the `.env` file. interpolate: Whether to interpolate variables using POSIX variable expansion. encoding: Encoding to be used to read the file. + execute_commands: Whether to execute `$(command)` substitutions in values. Returns: Bool: True if at least one environment variable is set else False @@ -431,6 +449,7 @@ def load_dotenv( interpolate=interpolate, override=override, encoding=encoding, + execute_commands=execute_commands, ) return dotenv.set_as_environment_variables() @@ -441,6 +460,7 @@ def dotenv_values( verbose: bool = False, interpolate: bool = True, encoding: Optional[str] = "utf-8", + execute_commands: bool = False, ) -> Dict[str, Optional[str]]: """ Parse a .env file and return its content as a dict. @@ -455,6 +475,7 @@ def dotenv_values( verbose: Whether to output a warning if the .env file is missing. interpolate: Whether to interpolate variables using POSIX variable expansion. encoding: Encoding to be used to read the file. + execute_commands: Whether to execute `$(command)` substitutions in values. If both `dotenv_path` and `stream` are `None`, `find_dotenv()` is used to find the .env file. @@ -469,6 +490,7 @@ def dotenv_values( interpolate=interpolate, override=True, encoding=encoding, + execute_commands=execute_commands, ).dict() diff --git a/src/dotenv/variables.py b/src/dotenv/variables.py index 667f2f26..c43eafb3 100644 --- a/src/dotenv/variables.py +++ b/src/dotenv/variables.py @@ -1,7 +1,13 @@ +import logging +import os import re +import subprocess from abc import ABCMeta, abstractmethod +from re import Match from typing import Iterator, Mapping, Optional, Pattern +logger = logging.getLogger(__name__) + _posix_variable: Pattern[str] = re.compile( r""" \$\{ @@ -13,6 +19,26 @@ """, re.VERBOSE, ) +_command: Pattern[str] = re.compile(r"\$\(([^)]+)\)") + + +def resolve_commands(value: str, env: Mapping[str, Optional[str]]) -> str: + cmd_env = {**os.environ, **{k: v for k, v in env.items() if v is not None}} + + def run(match: Match[str]) -> str: + try: + return subprocess.check_output( + match.group(1), + shell=True, + text=True, + stderr=subprocess.DEVNULL, + env=cmd_env, + ).strip() + except (subprocess.CalledProcessError, OSError): + logger.warning("python-dotenv: command failed: %s", match.group(1)) + return "" + + return _command.sub(run, value) class Atom(metaclass=ABCMeta): diff --git a/tests/test_cli.py b/tests/test_cli.py index d4e3ad4d..5f561d46 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -39,6 +39,17 @@ def test_list( assert (result.exit_code, result.output) == (0, expected) +def test_list_with_execute_commands(cli, dotenv_path): + dotenv_path.write_text("TOKEN=$(echo resolved)\n") + + result = cli.invoke( + dotenv_cli, ["--file", str(dotenv_path), "--execute-commands", "list"] + ) + + assert result.exit_code == 0 + assert result.output == "TOKEN=resolved\n" + + def test_list_non_existent_file(cli): result = cli.invoke(dotenv_cli, ["--file", "nx_file", "list"]) @@ -269,16 +280,16 @@ def test_run_with_command_flags(dotenv_path, tmp_path): """ Check that command flags passed after `dotenv run` are not interpreted. - Here, we want to run `printenv --version`, not `dotenv --version`. + Here, we want to run `python --version`, not `dotenv --version`. """ result = run_dotenv( - ["--file", str(dotenv_path), "run", "printenv", "--version"], + ["--file", str(dotenv_path), "run", "python", "--version"], cwd=tmp_path, ) check_process(result, exit_code=0) - assert result.stdout.strip().startswith("printenv ") + assert "Python" in result.stdout def test_run_with_dotenv_and_command_flags(dotenv_path, tmp_path): @@ -287,7 +298,7 @@ def test_run_with_dotenv_and_command_flags(dotenv_path, tmp_path): """ result = run_dotenv( - ["--version", "--file", str(dotenv_path), "run", "printenv", "--version"], + ["--version", "--file", str(dotenv_path), "run", "python", "--version"], cwd=tmp_path, ) diff --git a/tests/test_main.py b/tests/test_main.py index 6f9d4c5c..49a0282c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -756,3 +756,101 @@ def test_dotenv_values_empty_value_with_inline_comment(string, expected): result = dotenv.dotenv_values(stream=io.StringIO(string)) assert result == expected + + +@pytest.mark.skipif( + sys.platform == "win32", reason="This test assumes case-sensitive variable names" +) +@pytest.mark.parametrize( + "string,execute_commands,expected", + [ + ("TOKEN=$(echo abc)", False, {"TOKEN": "$(echo abc)"}), + ("TOKEN=$(echo abc)", True, {"TOKEN": "abc"}), + ('TOKEN="$(echo abc)"', True, {"TOKEN": "abc"}), + ("TOKEN='$(echo abc)'", True, {"TOKEN": "abc"}), + ("BASE=foo\nTOKEN=$(echo ${BASE})", True, {"BASE": "foo", "TOKEN": "foo"}), + ( + "BASE=foo\nPREFIX=${BASE}-$(echo suffix)", + True, + {"BASE": "foo", "PREFIX": "foo-suffix"}, + ), + ("TOKEN=$(false)", True, {"TOKEN": ""}), + ], +) +def test_dotenv_values_execute_commands(string, execute_commands, expected): + with mock.patch.dict(os.environ, {}, clear=True): + result = dotenv.dotenv_values( + stream=io.StringIO(string), + execute_commands=execute_commands, + ) + + assert result == expected + + +@pytest.mark.skipif( + sys.platform == "win32", reason="This test assumes case-sensitive variable names" +) +@mock.patch.dict(os.environ, {}, clear=True) +def test_dotenv_values_execute_commands_without_interpolate(): + result = dotenv.dotenv_values( + stream=io.StringIO("BASE=foo\nTOKEN=$(echo ${BASE})\nLITERAL=${BASE}"), + interpolate=False, + execute_commands=True, + ) + + assert result == {"BASE": "foo", "TOKEN": "foo", "LITERAL": "${BASE}"} + + +@pytest.mark.skipif( + sys.platform == "win32", reason="This test assumes case-sensitive variable names" +) +@mock.patch.dict(os.environ, {}, clear=True) +def test_dotenv_values_execute_commands_python(tmp_path): + helper = tmp_path / "helper.py" + helper.write_text('print("secret", end="")') + result = dotenv.dotenv_values( + stream=io.StringIO(f"TOKEN=$({sys.executable} {helper})"), + execute_commands=True, + ) + + assert result == {"TOKEN": "secret"} + + +@pytest.mark.skipif( + sys.platform == "win32", reason="This test assumes case-sensitive variable names" +) +@mock.patch.dict(os.environ, {}, clear=True) +def test_load_dotenv_execute_commands(dotenv_path): + dotenv_path.write_text("TOKEN=$(echo loaded)") + + result = dotenv.load_dotenv(dotenv_path, execute_commands=True) + + assert result is True + assert os.environ == {"TOKEN": "loaded"} + + +def test_load_dotenv_execute_commands_in_current_dir(tmp_path): + dotenv_path = tmp_path / ".env" + helper = tmp_path / "helper.py" + helper.write_text('print("from-subprocess", end="")') + dotenv_path.write_text(f"TOKEN=$({sys.executable} {helper})") + code_path = tmp_path / "code.py" + code_path.write_text( + textwrap.dedent(""" + import dotenv + import os + + dotenv.load_dotenv(execute_commands=True) + print(os.environ['TOKEN']) + """) + ) + os.chdir(tmp_path) + + result = subprocess.run( + [sys.executable, str(code_path)], + capture_output=True, + text=True, + check=True, + ) + + assert result.stdout == "from-subprocess\n" diff --git a/tests/test_parser.py b/tests/test_parser.py index 667eae15..ca64f7fc 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -715,6 +715,17 @@ ), ], ), + ( + "TOKEN=$(echo x)", + [ + Binding( + key="TOKEN", + value="$(echo x)", + original=Original(string="TOKEN=$(echo x)", line=1), + error=False, + ) + ], + ), ], ) def test_parse_stream(test_input, expected): diff --git a/tests/test_variables.py b/tests/test_variables.py index 6f2b2203..e198b790 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -1,6 +1,8 @@ +import sys + import pytest -from dotenv.variables import Literal, Variable, parse_variables +from dotenv.variables import Literal, Variable, parse_variables, resolve_commands @pytest.mark.parametrize( @@ -33,3 +35,28 @@ def test_parse_variables(value, expected): result = parse_variables(value) assert list(result) == expected + + +@pytest.mark.parametrize( + "value,env,expected", + [ + ("plain", {}, "plain"), + ("$(echo hello)", {}, "hello"), + ("prefix-$(echo suffix)", {}, "prefix-suffix"), + ("$(false)", {}, ""), + ("$(i_do_not_exist_xyz)", {}, ""), + ], +) +def test_resolve_commands(value, env, expected): + assert resolve_commands(value, env) == expected + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX shell variable syntax is not used on Windows" +) +def test_resolve_commands_shell_env_variable(): + assert resolve_commands("$(echo ${PREFIX})", {"PREFIX": "hi"}) == "hi" + + +def test_resolve_commands_strips_trailing_newline(): + assert resolve_commands("$(printf 'x\\n')", {}) == "x"