ietf-tools · rjsparks · Feb 19, 2025 · Nov 12, 2024 · Nov 13, 2024 · Nov 14, 2024
diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml
@@ -14,6 +14,10 @@ services:
             # - datatracker-vscode-ext:/root/.vscode-server/extensions
         # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
         network_mode: service:db
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
 
 volumes:
     datatracker-vscode-ext:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -28,6 +28,8 @@ jobs:
     services:
       db:
         image: ghcr.io/ietf-tools/datatracker-db:latest
+      blobstore:
+        image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
 
     steps:
     - uses: actions/checkout@v4

diff --git a/README.md b/README.md
@@ -106,6 +106,23 @@ Nightly database dumps of the datatracker are available as Docker images: `ghcr.
 
 > Note that to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script.
 
+### Blob storage for dev/test
+
+The dev and test environments use [minio](https://github.com/minio/minio) to provide local blob storage. See the settings files for how the app container communicates with the blobstore container. If you need to work with minio directly from outside the containers (to interact with its api or console), use `docker compose` from the top level directory of your clone to expose it at an ephemeral port.
+
+```
+$ docker compose port blobstore 9001
+0.0.0.0:<some ephemeral port>
+
+$ curl -I http://localhost:<some ephemeral port>
+HTTP/1.1 200 OK
+...
+```
+
+
+The minio container exposes the minio api at port 9000 and the minio console at port 9001
+
+
 ### Frontend Development
 
 #### Intro

diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
@@ -1,7 +1,9 @@
 # Copyright The IETF Trust 2007-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *   # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -79,3 +81,22 @@
 
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }
diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py
@@ -1,7 +1,9 @@
 # Copyright The IETF Trust 2007-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -66,3 +68,22 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }
diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml
@@ -28,5 +28,8 @@ services:
         volumes:
             - postgresdb-data:/var/lib/postgresql/data
 
+    blobstore:
+      image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
+
 volumes:
     postgresdb-data:
diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py
@@ -1,7 +1,9 @@
 # Copyright The IETF Trust 2007-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -65,3 +67,22 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -15,6 +15,7 @@ services:
         depends_on:
             - db
             - mq
+            - blobstore
 
         ipc: host
 
@@ -83,6 +84,14 @@ services:
             - .:/workspace
             - app-assets:/assets
 
+    blobstore:
+        image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
+        restart: unless-stopped
+        volumes:
+            - "minio-data:/data"
+
+
+
 #  Celery Beat is a periodic task runner. It is not normally needed for development,
 #  but can be enabled by uncommenting the following.
 #
@@ -106,3 +115,4 @@ services:
 volumes:
     postgresdb-data:
     app-assets:
+    minio-data:
diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile
@@ -43,8 +43,8 @@ RUN rm -rf /tmp/library-scripts
 # Copy the startup file
 COPY docker/scripts/app-init.sh /docker-init.sh
 COPY docker/scripts/app-start.sh /docker-start.sh
-RUN sed -i 's/\r$//' /docker-init.sh && chmod +x /docker-init.sh
-RUN sed -i 's/\r$//' /docker-start.sh && chmod +x /docker-start.sh
+RUN sed -i 's/\r$//' /docker-init.sh && chmod +rx /docker-init.sh
+RUN sed -i 's/\r$//' /docker-start.sh && chmod +rx /docker-start.sh
 
 # Fix user UID / GID to match host
 RUN groupmod --gid $USER_GID $USERNAME \

diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
@@ -1,11 +1,13 @@
-# Copyright The IETF Trust 2007-2019, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
-from ietf.settings_postgresqldb import DATABASES   # pyflakes:ignore
+from ietf.settings_postgresqldb import DATABASES  # pyflakes:ignore
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
@@ -37,6 +39,25 @@
 # DEV_TEMPLATE_CONTEXT_PROCESSORS = [
 #    'ietf.context_processors.sql_debug',
 # ]
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=storagename,
+        ),
+    }
+
 
 DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/'
 INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/'

diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml
@@ -16,6 +16,10 @@ services:
     pgadmin:
         ports:
             - '5433'
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
     celery:
         volumes:
             - .:/workspace

diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+import boto3
+import os
+import sys
+
+from ietf.settings import MORE_STORAGE_NAMES
+
+
+def init_blobstore():
+    blobstore = boto3.resource(
+        "s3",
+        endpoint_url=os.environ.get("BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000"),
+        aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"),
+        aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"),
+        aws_session_token=None,
+        config=botocore.config.Config(signature_version="s3v4"),
+        verify=False,
+    )
+    for bucketname in MORE_STORAGE_NAMES:
+        blobstore.create_bucket(
+            Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(init_blobstore())
diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh
@@ -73,6 +73,11 @@ echo "Creating data directories..."
 chmod +x ./docker/scripts/app-create-dirs.sh
 ./docker/scripts/app-create-dirs.sh
 
+# Configure the development blobstore
+
+echo "Configuring blobstore..."
+PYTHONPATH=/workspace python ./docker/scripts/app-configure-blobstore.py
+
 # Download latest coverage results file
 
 echo "Downloading latest coverage results file..."

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
@@ -25,6 +25,7 @@
 import debug                            # pyflakes:ignore
 
 import ietf
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils import get_unicode_document_content
 from ietf.doc.models import RelatedDocument, State
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
@@ -553,6 +554,10 @@ def test_api_upload_polls_and_chatlog(self):
             newdoc = session.presentations.get(document__type_id=type_id).document
             newdoccontent = get_unicode_document_content(newdoc.name, Path(session.meeting.get_materials_path()) / type_id / newdoc.uploaded_filename)
             self.assertEqual(json.loads(content), json.loads(newdoccontent))
+            self.assertEqual(
+                json.loads(retrieve_str(type_id, newdoc.uploaded_filename)),
+                json.loads(content)
+            )
 
     def test_api_upload_bluesheet(self):
         url = urlreverse("ietf.meeting.views.api_upload_bluesheet")

diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
@@ -12,7 +12,7 @@
     TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent,
     AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL,
     ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder,
-    BofreqEditorDocEvent, BofreqResponsibleDocEvent )
+    BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject )
 
 from ietf.utils.validators import validate_external_resource_value
 
@@ -218,3 +218,9 @@ class DocExtResourceAdmin(admin.ModelAdmin):
     search_fields = ['doc__name', 'value', 'display_name', 'name__slug',]
     raw_id_fields = ['doc', ]
 admin.site.register(DocExtResource, DocExtResourceAdmin)
+
+class StoredObjectAdmin(admin.ModelAdmin):
+    list_display = ['store', 'name', 'modified', 'deleted']
+    list_filter = ['deleted']
+    search_fields = ['store', 'name', 'doc_name', 'doc_rev', 'deleted']
+admin.site.register(StoredObject, StoredObjectAdmin)
diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
@@ -13,6 +13,7 @@
 
 from typing import List, Optional      # pyflakes:ignore
 
+from ietf.doc.storage_utils import exists_in_storage, remove_from_storage
 from ietf.doc.utils import update_action_holders
 from ietf.utils import log
 from ietf.utils.mail import send_mail
@@ -156,11 +157,17 @@ def remove_ftp_copy(f):
         if mark.exists():
             mark.unlink()
 
+    def remove_from_active_draft_storage(file):
+        # Assumes the glob will never find a file with no suffix
+        ext = file.suffix[1:]
+        remove_from_storage("active-draft", f"{ext}/{file.name}", warn_if_missing=False)
 
+    # Note that the object is already in the "draft" storage.
     src_dir = Path(settings.INTERNET_DRAFT_PATH)
     for file in src_dir.glob("%s-%s.*" % (doc.name, rev)):
         move_file(str(file.name))
         remove_ftp_copy(str(file.name))
+        remove_from_active_draft_storage(file)
 
 def expire_draft(doc):
     # clean up files
@@ -218,6 +225,13 @@ def move_file_to(subdir):
             mark = Path(settings.FTP_DIR) / "internet-drafts" / basename
             if mark.exists():
                 mark.unlink()
+            if ext:
+                # Note that we're not moving these strays anywhere - the assumption
+                # is that the active-draft blobstore will not get strays.
+                # See, however, the note about "major system failures" at "unknown_ids"
+                blobname = f"{ext[1:]}/{basename}"
+                if exists_in_storage("active-draft", blobname):
+                    remove_from_storage("active-draft", blobname)
 
         try:
             doc = Document.objects.get(name=filename, rev=revision)