8000 Environment variable fixes (redun init and db creds) by mattrasmus · Pull Request #123 · insitro/redun · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Environment variable fixes (redun init and db creds) #123

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
May 29, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions redun/backends/db/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3144,18 +3144,31 @@ def _get_uri_from_secret(secret_name: str) -> str:
def _get_credentialed_uri(base_uri: str, conf: Section) -> str:
parts = urlparse(base_uri)

if "@" in parts.netloc:
if parts.password:
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Before we didn't allow even a username to be passed. Usernames are fine. Instead let's just test for the presence of passwords, which is what we really care about.

raise RedunDatabaseError(
"rejected db_uri containing credentials. use environment variables instead"
f"Do not include passwords in DB URIs. Use environment variables instead (e.g {DEFAULT_DB_USERNAME_ENV} and {DEFAULT_DB_PASSWORD_ENV})"
)

credentials = RedunBackendDb._get_login_credentials(conf)
credentials = RedunBackendDb._get_login_credentials(conf, parts.username)
if credentials:
parts = parts._replace(netloc=f"{credentials}@{parts.netloc}")
if "@" in parts.netloc:
_, netloc = parts.netloc.rsplit("@", 1)
else:
netloc = parts.netloc
parts = parts._replace(netloc=f"{credentials}@{netloc}")
return urlunparse(parts)

@staticmethod
def _get_login_credentials(conf: Section) -> Optional[str]:
user = os.getenv(conf.get("db_username_env", DEFAULT_DB_USERNAME_ENV))
def _get_login_credentials(conf: Section, username: Optional[str] = None) -> Optional[str]:
# Replace username by env var if defined.
username = os.getenv(conf.get("db_username_env", DEFAULT_DB_USERNAME_ENV), username)

password = quote_plus(os.getenv(conf.get("db_password_env", DEFAULT_DB_PASSWORD_ENV), ""))
return user and password and f"{user}:{password}"

if username:
if password:
return f"{username}:{password}"
else:
return username
else:
return None
10 changes: 4 additions & 6 deletions redun/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1463,13 +1463,11 @@ def init_command(self, args: Namespace, extra_args: List[str], argv: List[str])
"""
Initialize redun project directory.
"""
if not args.config:
if extra_args:
basedir = extra_args[0]
else:
basedir = "."
if extra_args:
basedir = extra_args[0]
args.config = os.path.join(basedir, REDUN_CONFIG_DIR)

else:
args.config = get_config_dir(args.config)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using get_config_dir() makes use of all usual ways of discovering the config dir (env var, args, etc).

self.get_scheduler(args, migrate=True)
self.display("Initialized redun repository: {}".format(args.config))

Expand Down
31 changes: 31 additions & 0 deletions redun/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,37 @@ def main(x: File, y: List[File]):
assert "--y Y" in output


@use_tempdir
def test_init() -> None:
"""
Ensure `redun init` creates a new redun config dir.
"""
# Default config dir.
client = RedunClient()
client.execute(["redun", "init"])
assert File(".redun/redun.ini").exists()
assert File(".redun/redun.db").exists()

# Config dir specified by flag.
client = RedunClient()
client.execute(["redun", "--config", "config1", "init"])
assert File("config1/redun.ini").exists()
assert File("config1/redun.db").exists()

# Config dir specified by positional argument.
client = RedunClient()
client.execute(["redun", "init", "dir1"])
assert File("dir1/.redun/redun.ini").exists()
assert File("dir1/.redun/redun.db").exists()

# Config dir specified by env var.
with patch.dict(os.environ, {"REDUN_CONFIG": "config2"}, clear=True):
client = RedunClient()
client.execute(["redun", "init"])
assert File("config2/redun.ini").exists()
assert File("config2/redun.db").exists()


@use_tempdir
@patch.dict(os.environ, {AWS_ARRAY_VAR: "1"}, clear=True)
def test_oneshot_existing_output() -> None:
Expand Down
24 changes: 24 additions & 0 deletions redun/tests/test_uri_parsing.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import os
import json
from unittest.mock import patch

Expand Down Expand Up @@ -57,6 +58,29 @@ def test_uri_with_credentials() -> None:
RDB._get_uri("postgresql://user:secret@localhost:5432/redun", {})


@patch.dict(os.environ, {"REDUN_DB_USERNAME": "", "REDUN_DB_PASSWORD": ""}, clear=True)
def test_uri_with_username() -> None:
# Only specifying username should be allowed.
assert (
RDB._get_uri("postgresql://user@localhost:5432/redun", {})
== "postgresql://user@localhost:5432/redun"
)

# Env var should override username.
with patch.dict(os.environ, {"REDUN_DB_USERNAME": "alice"}, clear=True):
assert (
RDB._get_uri("postgresql://user@localhost:5432/redun", {})
== "postgresql://alice@localhost:5432/redun"
)

# Username env var alone should be allowed.
with patch.dict(os.environ, {"REDUN_DB_USERNAME": "alice"}, clear=True):
assert (
RDB._get_uri("postgresql://localhost:5432/redun", {})
== "postgresql://alice@localhost:5432/redun"
)


@patch("redun.executors.aws_utils.get_aws_client")
def test_uri_from_secret(client_mock) -> None:
engine = "postgres"
Expand Down
0