From cadc4667c8b25ef98ce029081d77e9845dcd1a1f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 16:56:31 -0400 Subject: [PATCH 1/2] feat: add timeout / retry limit to boto client --- dev/deploy-to-container/settings_local.py | 7 ++++++- dev/diff/settings_local.py | 7 ++++++- dev/tests/settings_local.py | 7 ++++++- docker/configs/settings_local.py | 7 ++++++- ietf/settings.py | 4 ++++ ietf/settings_test.py | 7 ++++++- k8s/settings_local.py | 7 ++++++- 7 files changed, 40 insertions(+), 6 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index df0e7cc7f1..1e105dd721 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -89,7 +89,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index b0994dcfbf..c8e327115e 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -76,7 +76,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index b8815b837b..72caad60a9 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -75,7 +75,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 8771053419..019caeba9c 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -46,7 +46,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=storagename, ), diff --git a/ietf/settings.py b/ietf/settings.py index 256603a46d..83c408e656 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -186,6 +186,10 @@ ENABLE_BLOBSTORAGE = True +BLOBSTORAGE_MAX_ATTEMPTS = 1 +BLOBSTORAGE_CONNECT_TIMEOUT = 2 +BLOBSTORAGE_READ_TIMEOUT = 2 + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index a86c98a5fb..ba258570bf 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -122,7 +122,12 @@ def tempdir_with_cleanup(**kwargs): access_key=_blob_store_access_key, secret_key=_blob_store_secret_key, security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), bucket_name=f"{_blob_store_bucket_prefix}{storagename}", ietf_log_blob_timing=_blob_store_enable_profiling, ), diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 9761ddf57c..42d4e0c3ce 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -330,7 +330,12 @@ def _multiline_to_list(s): access_key=_blob_store_access_key, secret_key=_blob_store_secret_key, security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), ietf_log_blob_timing=_blob_store_enable_profiling, ), From 9371a5f52bb2724ab43423e9c9d66d56d0f88588 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 17:00:42 -0400 Subject: [PATCH 2/2] ci: let k8s config the timeouts via env --- k8s/settings_local.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 42d4e0c3ce..1ecd77a5d5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -315,6 +315,15 @@ def _multiline_to_list(s): _blob_store_enable_profiling = ( os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" ) +_blob_store_max_attempts = ( + os.environ.get("DATATRACKER_BLOB_STORE_MAX_ATTEMPTS", BLOBSTORAGE_MAX_ATTEMPTS) +) +_blob_store_connect_timeout = ( + os.environ.get("DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT) +) +_blob_store_read_timeout = ( + os.environ.get("DATATRACKER_BLOB_STORE_READ_TIMEOUT", BLOBSTORAGE_READ_TIMEOUT) +) try: from ietf.settings import MORE_STORAGE_NAMES except ImportError: @@ -332,9 +341,9 @@ def _multiline_to_list(s): security_token=None, client_config=boto3.session.Config( signature_version="s3v4", - connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, - read_timeout=BLOBSTORAGE_READ_TIMEOUT, - retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + connect_timeout=_blob_store_connect_timeout, + read_timeout=_blob_store_read_timeout, + retries={"total_max_attempts": _blob_store_max_attempts}, ), bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), ietf_log_blob_timing=_blob_store_enable_profiling,