From 328f3b5b3852e2b9565e81ab55d5f4c3e2b669cc Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 12 Nov 2024 09:44:01 -0600 Subject: [PATCH 01/87] feat: basic blobstore infrastructure for dev --- .devcontainer/devcontainer.json | 10 +++++++++- .devcontainer/docker-compose.extend.yml | 2 ++ docker-compose.yml | 15 +++++++++++++++ docker/docker-compose.extend.yml | 4 ++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ac7854f265..3708d868ac 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -60,7 +60,7 @@ }, // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [3000, 5432, 8000], + "forwardPorts": [3000, 5432, 8000, 9000, 9001], "portsAttributes": { "3000": { @@ -78,6 +78,14 @@ "8001": { "label": "Datatracker", "onAutoForward": "ignore" + }, + "9000": { + "label": "Minio", + "onAutoForward": "silent" + }, + "9001": { + "label": "Minio Console", + "onAutoForward": "silent" } }, diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index fa9a412cf2..075e618778 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,6 +14,8 @@ services: # - datatracker-vscode-ext:/root/.vscode-server/extensions # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db + blobstore: + network_mode: service:db volumes: datatracker-vscode-ext: diff --git a/docker-compose.yml b/docker-compose.yml index 65b28f54fe..89faee5ebf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: depends_on: - db - mq + - blobstore ipc: host @@ -83,6 +84,19 @@ services: - .:/workspace - app-assets:/assets + blobstore: + image: quay.io/minio/minio:latest + restart: unless-stopped + volumes: + - "minio-data:/data" + environment: + - MINIO_ROOT_USER=minio_root + - MINIO_ROOT_PASSWORD=minio_pass + - MINIO_DEFAULT_BUCKETS=defaultbucket + command: server --console-address ":9001" /data + + + # Celery Beat is a periodic task runner. It is not normally needed for development, # but can be enabled by uncommenting the following. # @@ -105,3 +119,4 @@ services: volumes: postgresdb-data: app-assets: + minio-data: diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index 0538c0d3e9..a69a453110 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -16,6 +16,10 @@ services: pgadmin: ports: - '5433' + blobstore: + ports: + - '9000' + - '9001' celery: volumes: - .:/workspace From 5d5aea10552a220a84ab31221fc6d686e0e59fba Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 13 Nov 2024 12:53:48 -0600 Subject: [PATCH 02/87] refactor: (broken) attempt to put minio console behind nginx --- .devcontainer/docker-compose.extend.yml | 4 ++-- docker/configs/nginx-proxy.conf | 26 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index 075e618778..83d2672a7f 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,8 +14,8 @@ services: # - datatracker-vscode-ext:/root/.vscode-server/extensions # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db - blobstore: - network_mode: service:db + #blobstore: + # network_mode: service:db volumes: datatracker-vscode-ext: diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index 3068cc71d7..066ca214bf 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -21,6 +21,32 @@ server { proxy_redirect off; } + location /blobstore/ { + rewrite ^/blobstore/(.*) /$1 break; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websockets in MinIO versions released after January 2023 + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + # Some environments may encounter CORS errors (Kubernetes + Nginx Ingress) + # Uncomment the following line to set the Origin request to an empty string + proxy_set_header Origin ''; + + chunked_transfer_encoding off; + + proxy_pass http://blobstore:9001; + } + location / { error_page 502 /502.html; proxy_pass http://localhost:8001/; From be13bd6d41e180a39c44e4ae5534248a3cface75 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 14 Nov 2024 16:23:17 -0600 Subject: [PATCH 03/87] feat: initialize blobstore with boto3 --- docker/app.Dockerfile | 4 ++-- docker/scripts/app-configure-blobstore.py | 19 +++++++++++++++++++ docker/scripts/app-init.sh | 5 +++++ requirements.txt | 3 ++- 4 files changed, 28 insertions(+), 3 deletions(-) create mode 100755 docker/scripts/app-configure-blobstore.py diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index b7dd44b6f1..fee3833733 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -43,8 +43,8 @@ RUN rm -rf /tmp/library-scripts # Copy the startup file COPY docker/scripts/app-init.sh /docker-init.sh COPY docker/scripts/app-start.sh /docker-start.sh -RUN sed -i 's/\r$//' /docker-init.sh && chmod +x /docker-init.sh -RUN sed -i 's/\r$//' /docker-start.sh && chmod +x /docker-start.sh +RUN sed -i 's/\r$//' /docker-init.sh && chmod +rx /docker-init.sh +RUN sed -i 's/\r$//' /docker-start.sh && chmod +rx /docker-start.sh # Fix user UID / GID to match host RUN groupmod --gid $USER_GID $USERNAME \ diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py new file mode 100755 index 0000000000..a100a52d2b --- /dev/null +++ b/docker/scripts/app-configure-blobstore.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# Copyright The IETF Trust 2024, All Rights Reserved + +import boto3 +import sys + +def init_blobstore(): + blobstore = boto3.resource("s3", + endpoint_url="http://blobstore:9000", + aws_access_key_id="minio_root", + aws_secret_access_key="minio_pass", + aws_session_token=None, + config=boto3.session.Config(signature_version="s3v4"), + verify=False + ) + blobstore.create_bucket(Bucket="ietfdata") + +if __name__ == "__main__": + sys.exit(init_blobstore()) diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index b96b88f1f5..a3e405560f 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -73,6 +73,11 @@ echo "Creating data directories..." chmod +x ./docker/scripts/app-create-dirs.sh ./docker/scripts/app-create-dirs.sh +# Configure the development blobstore + +echo "Configuring blobstore..." +python ./docker/scripts/app-configure-blobstore.py + # Download latest coverage results file echo "Downloading latest coverage results file..." diff --git a/requirements.txt b/requirements.txt index 2e6e2714d7..4cd9ee6bcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ beautifulsoup4>=4.11.1 # Only used in tests bibtexparser>=1.2.0 # Only used in tests bleach>=6 types-bleach>=6 +boto3>=1.35 celery>=5.2.6 coverage>=4.5.4,<5.0 # Coverage 5.x moves from a json database to SQLite. Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views defusedxml>=0.7.1 # for TastyPie when using xml; not a declared dependency @@ -71,7 +72,7 @@ tblib>=1.7.0 # So that the django test runner provides tracebacks tlds>=2022042700 # Used to teach bleach about which TLDs currently exist tqdm>=4.64.0 Unidecode>=1.3.4 -urllib3>=2 +urllib3>=1.26,<2 weasyprint>=59 xml2rfc[pdf]>=3.23.0 xym>=0.6,<1.0 From 9999edcfe0cebfdc3658559aa10d06fde8e38b48 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 18 Nov 2024 14:14:40 -0600 Subject: [PATCH 04/87] fix: abandon attempt to proxy minio. Use docker compose instead. --- .devcontainer/devcontainer.json | 10 +--------- .devcontainer/docker-compose.extend.yml | 6 ++++-- README.md | 17 ++++++++++++++++ docker/configs/nginx-proxy.conf | 26 ------------------------- 4 files changed, 22 insertions(+), 37 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3708d868ac..ac7854f265 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -60,7 +60,7 @@ }, // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [3000, 5432, 8000, 9000, 9001], + "forwardPorts": [3000, 5432, 8000], "portsAttributes": { "3000": { @@ -78,14 +78,6 @@ "8001": { "label": "Datatracker", "onAutoForward": "ignore" - }, - "9000": { - "label": "Minio", - "onAutoForward": "silent" - }, - "9001": { - "label": "Minio Console", - "onAutoForward": "silent" } }, diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index 83d2672a7f..286eefb29c 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,8 +14,10 @@ services: # - datatracker-vscode-ext:/root/.vscode-server/extensions # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db - #blobstore: - # network_mode: service:db + blobstore: + ports: + - '9000' + - '9001' volumes: datatracker-vscode-ext: diff --git a/README.md b/README.md index ee9865ba21..0ece0eb03b 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,23 @@ Nightly database dumps of the datatracker are available as Docker images: `ghcr. > Note that to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script. +### Blob storage for dev/test + +The dev and test environments use [minio](https://github.com/minio/minio) to provide local blob storage. See the settings files for how the app container communicates with the blobstore container. If you need to work with minio directly from outside the containers (to interact with its api or console), use `docker compose` from the top level directory of your clone to expose it at an ephemeral port. + +``` +$ docker compose port blobstore 9001 +0.0.0.0: + +$ curl -I http://localhost: +HTTP/1.1 200 OK +... +``` + + +The minio container exposes the minio api at port 9000 and the minio console at port 9001 + + ### Frontend Development #### Intro diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index 066ca214bf..3068cc71d7 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -21,32 +21,6 @@ server { proxy_redirect off; } - location /blobstore/ { - rewrite ^/blobstore/(.*) /$1 break; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-NginX-Proxy true; - - # This is necessary to pass the correct IP to be hashed - real_ip_header X-Real-IP; - - proxy_connect_timeout 300; - - # To support websockets in MinIO versions released after January 2023 - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - # Some environments may encounter CORS errors (Kubernetes + Nginx Ingress) - # Uncomment the following line to set the Origin request to an empty string - proxy_set_header Origin ''; - - chunked_transfer_encoding off; - - proxy_pass http://blobstore:9001; - } - location / { error_page 502 /502.html; proxy_pass http://localhost:8001/; From 6fc22416ae070ca032c2fa257f444a66e0d8150f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 15 Jan 2025 12:44:28 -0600 Subject: [PATCH 05/87] feat: beginning of blob writes --- docker/configs/settings_local.py | 20 +++++++++++--- docker/scripts/app-configure-blobstore.py | 33 ++++++++++++++++++++++- ietf/doc/tests_bofreq.py | 3 +++ ietf/doc/tests_charter.py | 6 +++++ ietf/doc/tests_conflict_review.py | 2 ++ ietf/doc/views_bofreq.py | 3 +++ ietf/doc/views_charter.py | 9 ++++--- ietf/doc/views_conflict_review.py | 7 +++-- ietf/settings.py | 6 +++++ ietf/settings_test.py | 23 +++++++++++++++- ietf/submit/tests.py | 2 ++ ietf/submit/utils.py | 4 +++ requirements.txt | 1 + 13 files changed, 109 insertions(+), 10 deletions(-) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 5df5d15e82..1b458324e1 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -1,11 +1,12 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- -from ietf.settings import * # pyflakes:ignore +from ietf.settings import * # pyflakes:ignore +import boto3 ALLOWED_HOSTS = ['*'] -from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore +from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/" @@ -37,6 +38,19 @@ # DEV_TEMPLATE_CONTEXT_PROCESSORS = [ # 'ietf.context_processors.sql_debug', # ] +STORAGES["ietfdata"] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url="http://blobstore:9000", + access_key="minio_root", + secret_key="minio_pass", + security_token=None, + client_config=boto3.session.Config(signature_version="s3v4"), + verify=False, + bucket_name="ietfdata", + ), +} + DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/' INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/' diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index a100a52d2b..40202f6fbc 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -13,7 +13,38 @@ def init_blobstore(): config=boto3.session.Config(signature_version="s3v4"), verify=False ) - blobstore.create_bucket(Bucket="ietfdata") + for bucketname in [ + "agenda", + "bluesheets", + "bofreq", + "charter", + "chatlog", + "conflrev", + "draft-xml", + "draft-txt", + "draft-html", + "draft-htmlized", + "draft-pdf", + "draft-pdfized", + "liai-att", + "liaison", + "minutes", + "narrativeminutes", + "polls", + "procmaterials", + "recording", + "review", + "rfc-txt", + "rfc-html", + "rfc-pdf", + "rfc-htmlized", + "rfc-pdfized", + "slides", + "statchg", + "statement", + ]: + blobstore.create_bucket(Bucket=bucketname) + if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py index 2e27efd627..6a7c9393ef 100644 --- a/ietf/doc/tests_bofreq.py +++ b/ietf/doc/tests_bofreq.py @@ -16,6 +16,7 @@ from django.template.loader import render_to_string from django.utils import timezone +from ietf.doc.storage_utils import retrieve_str from ietf.group.factories import RoleFactory from ietf.doc.factories import BofreqFactory, NewRevisionDocEventFactory from ietf.doc.models import State, Document, NewRevisionDocEvent @@ -340,6 +341,7 @@ def test_submit(self): doc = reload_db_objects(doc) self.assertEqual('%02d'%(int(rev)+1) ,doc.rev) self.assertEqual(f'# {username}', doc.text()) + self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name())) self.assertEqual(docevent_count+1, doc.docevent_set.count()) self.assertEqual(1, len(outbox)) rev = doc.rev @@ -379,6 +381,7 @@ def test_start_new_bofreq(self): self.assertEqual(list(bofreq_editors(bofreq)), [nobody]) self.assertEqual(bofreq.latest_event(NewRevisionDocEvent).rev, '00') self.assertEqual(bofreq.text_or_error(), 'some stuff') + self.assertEqual(retrieve_str('bofreq',bofreq.get_base_name()), 'some stuff') self.assertEqual(len(outbox),1) finally: os.unlink(file.name) diff --git a/ietf/doc/tests_charter.py b/ietf/doc/tests_charter.py index e0207fe842..62e49559e2 100644 --- a/ietf/doc/tests_charter.py +++ b/ietf/doc/tests_charter.py @@ -16,6 +16,7 @@ from ietf.doc.factories import CharterFactory, NewRevisionDocEventFactory, TelechatDocEventFactory from ietf.doc.models import ( Document, State, BallotDocEvent, BallotType, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent ) +from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import ( next_revision, default_review_text, default_action_text, charter_name_for_group ) from ietf.doc.utils import close_open_ballots @@ -519,6 +520,11 @@ def test_submit_charter(self): ftp_charter_path = Path(settings.FTP_DIR) / "charter" / charter_path.name self.assertTrue(ftp_charter_path.exists()) self.assertTrue(charter_path.samefile(ftp_charter_path)) + blobstore_contents = retrieve_str("charter", charter.get_base_name()) + self.assertEqual( + blobstore_contents, + "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode("utf-8"), + ) def test_submit_initial_charter(self): diff --git a/ietf/doc/tests_conflict_review.py b/ietf/doc/tests_conflict_review.py index d2f94922b2..791db17f5a 100644 --- a/ietf/doc/tests_conflict_review.py +++ b/ietf/doc/tests_conflict_review.py @@ -16,6 +16,7 @@ from ietf.doc.factories import IndividualDraftFactory, ConflictReviewFactory, RgDraftFactory from ietf.doc.models import Document, DocEvent, NewRevisionDocEvent, BallotPositionDocEvent, TelechatDocEvent, State, DocTagName +from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils import create_ballot_if_not_open from ietf.doc.views_conflict_review import default_approval_text from ietf.group.models import Person @@ -422,6 +423,7 @@ def test_initial_submission(self): f.close() self.assertTrue(ftp_path.exists()) self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc) + self.assertEqual(retrieve_str("conflrev",basename), "Some initial review text\n") def test_subsequent_submission(self): doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index 3bd10287b2..e52dd2c56f 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -17,6 +17,7 @@ email_bofreq_new_revision, email_bofreq_responsible_changed) from ietf.doc.models import (Document, DocEvent, NewRevisionDocEvent, BofreqEditorDocEvent, BofreqResponsibleDocEvent, State) +from ietf.doc.storage_utils import store_str from ietf.doc.utils import add_state_change_event from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible from ietf.ietfauth.utils import has_role, role_required @@ -101,6 +102,7 @@ def submit(request, name): content = form.cleaned_data['bofreq_content'] with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination: destination.write(content) + store_str("bofreq", bofreq.get_base_name(), content) email_bofreq_new_revision(request, bofreq) return redirect('ietf.doc.views_doc.document_main', name=bofreq.name) @@ -175,6 +177,7 @@ def new_bof_request(request): content = form.cleaned_data['bofreq_content'] with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination: destination.write(content) + store_str("bofreq", bofreq.get_base_name(), content) email_bofreq_new_revision(request, bofreq) return redirect('ietf.doc.views_doc.document_main', name=bofreq.name) diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index f8748d2126..74a3c0124d 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -26,6 +26,7 @@ from ietf.doc.models import ( Document, DocHistory, State, DocEvent, BallotDocEvent, BallotPositionDocEvent, InitialReviewDocEvent, NewRevisionDocEvent, WriteupDocEvent, TelechatDocEvent ) +from ietf.doc.storage_utils import store_str from ietf.doc.utils import ( add_state_change_event, close_open_ballots, create_ballot, get_chartering_type ) from ietf.doc.utils_charter import ( historic_milestones_for_charter, @@ -441,9 +442,10 @@ def submit(request, name, option=None): ) # update rev with charter_filename.open("w", encoding="utf-8") as destination: if form.cleaned_data["txt"]: - destination.write(form.cleaned_data["txt"]) + content=form.cleaned_data["txt"] else: - destination.write(form.cleaned_data["content"]) + content=form.cleaned_data["content"] + destination.write(content) # Also provide a copy to the legacy ftp source directory, which is served by rsync # This replaces the hardlink copy that ghostlink has made in the past # Still using a hardlink as long as these are on the same filesystem. @@ -454,7 +456,8 @@ def submit(request, name, option=None): log( "There was an error creating a hardlink at %s pointing to %s" % (ftp_filename, charter_filename) - ) + ) + store_str("charter", charter_filename.name, content) if option in ["initcharter", "recharter"] and charter.ad == None: diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py index e55661ccdf..eb26fff5e7 100644 --- a/ietf/doc/views_conflict_review.py +++ b/ietf/doc/views_conflict_review.py @@ -19,6 +19,7 @@ from ietf.doc.models import ( BallotDocEvent, BallotPositionDocEvent, DocEvent, Document, NewRevisionDocEvent, State ) +from ietf.doc.storage_utils import store_str from ietf.doc.utils import ( add_state_change_event, close_open_ballots, create_ballot_if_not_open, update_telechat ) from ietf.doc.mails import email_iana, email_ad_approved_conflict_review @@ -186,9 +187,10 @@ def save(self, review): filepath = Path(settings.CONFLICT_REVIEW_PATH) / basename with filepath.open('w', encoding='utf-8') as destination: if self.cleaned_data['txt']: - destination.write(self.cleaned_data['txt']) + content = self.cleaned_data['txt'] else: - destination.write(self.cleaned_data['content']) + content = self.cleaned_data['content'] + destination.write(content) ftp_filepath = Path(settings.FTP_DIR) / "conflict-reviews" / basename try: os.link(filepath, ftp_filepath) # Path.hardlink_to is not available until 3.10 @@ -197,6 +199,7 @@ def save(self, review): "There was an error creating a hardlink at %s pointing to %s: %s" % (ftp_filepath, filepath, e) ) + store_str("conflrev", basename, content) #This is very close to submit on charter - can we get better reuse? @role_required('Area Director','Secretariat') diff --git a/ietf/settings.py b/ietf/settings.py index b452864be6..8819baa03d 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -6,6 +6,7 @@ # BASE_DIR and "settings_local" are from # http://code.djangoproject.com/wiki/SplitSettings +import boto3 # pyflakes:ignore import os import sys import datetime @@ -735,6 +736,11 @@ def skip_unreadable_post(record): "schedule_name": r"(?P[A-Za-z0-9-:_]+)", } +STORAGES = { + "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"}, + "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}, +} + # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 94ca22c71b..20fd9d8993 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import TEST_CODE_COVERAGE_CHECKER +from ietf.settings import boto3, STORAGES, TEST_CODE_COVERAGE_CHECKER import debug # pyflakes:ignore debug.debug = True @@ -105,3 +105,24 @@ def tempdir_with_cleanup(**kwargs): 'level': 'INFO', }, } + +for bucketname in [ + "bofreq", + "charter", + "conflrev", + "draft-xml", + "draft-txt", + "draft-html", +]: + STORAGES[bucketname] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url="http://blobstore:9000", + access_key="minio_root", + secret_key="minio_pass", + security_token=None, + client_config=boto3.session.Config(signature_version="s3v4"), + verify=False, + bucket_name=bucketname, + ), + } diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 6a56839177..5ad4e3b4f0 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -31,6 +31,7 @@ ReviewFactory, WgRfcFactory) from ietf.doc.models import ( Document, DocEvent, State, BallotPositionDocEvent, DocumentAuthor, SubmissionDocEvent ) +from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils import create_ballot_if_not_open, can_edit_docextresources, update_action_holders from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group @@ -428,6 +429,7 @@ def submit_new_wg(self, formats): self.assertTrue(draft.latest_event(type="added_suggested_replaces")) self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))) self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev)))) + self.assertTrue(len(retrieve_str("draft-txt",f"{name}-{rev}.txt"))>0) self.assertEqual(draft.type_id, "draft") self.assertEqual(draft.stream_id, "ietf") self.assertTrue(draft.expires >= timezone.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1)) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 16cccc9b59..1fb9d4f6b4 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -36,6 +36,7 @@ DocumentAuthor, AddedMessageEvent ) from ietf.doc.models import NewRevisionDocEvent from ietf.doc.models import RelatedDocument, DocRelationshipName, DocExtResource +from ietf.doc.storage_utils import store_bytes from ietf.doc.utils import (add_state_change_event, rebuild_reference_relations, set_replaces_for_document, prettify_std_name, update_doc_extresources, can_edit_docextresources, update_documentauthors, update_action_holders, @@ -665,6 +666,9 @@ def move_files_to_repository(submission): ftp_dest = Path(settings.FTP_DIR) / "internet-drafts" / dest.name os.link(dest, all_archive_dest) os.link(dest, ftp_dest) + with open(dest,"rb") as f: + content_bytes = f.read() + store_bytes(f"draft-{ext}", fname, content_bytes) elif dest.exists(): log.log("Intended to move '%s' to '%s', but found source missing while destination exists.") elif f".{ext}" in submission.file_types.split(','): diff --git a/requirements.txt b/requirements.txt index 7f2822bfdf..1431ea1441 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ django-markup>=1.5 # Limited use - need to reconcile against direct use of ma django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return django-referrer-policy>=1.0 django-simple-history>=3.0.0 +django-storages>=1.14.4 django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below django-tastypie>=0.14.7,<0.15.0 # Version must be locked in sync with version of Django django-vite>=2.0.2,<3 From 658dd7c128b03f7a384964d6c76e35c6212d35b5 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 15 Jan 2025 12:49:02 -0600 Subject: [PATCH 06/87] feat: storage utilities --- ietf/doc/storage_utils.py | 64 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 ietf/doc/storage_utils.py diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py new file mode 100644 index 0000000000..9cf336428e --- /dev/null +++ b/ietf/doc/storage_utils.py @@ -0,0 +1,64 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import debug # pyflakes ignore + +from django.core.files.base import ContentFile +from django.core.files.storage import storages, Storage + +from ietf.utils.log import log + +def _get_storage(kind: str) -> Storage: + if kind in [ + "bofreq", + "charter", + "conflrev", + "draft-xml", + "draft-txt", + "draft-html", + ]: + return storages[kind] + else: + debug.say(f"Got into not-implemented looking for {kind}") + raise NotImplementedError(f"Don't know how to store {kind}") + + +def store_bytes(kind: str, name: str, content: bytes, allow_overwrite: bool = False) -> None: + store = _get_storage(kind) + if not allow_overwrite: + try: + new_name = store.save(name, ContentFile(content)) + except Exception as e: + # Log and then swallow the exception while we're learning. + # Don't let failure pass so quietly when these are the autoritative bits. + log(f"Failed to save {kind}:{name}", e) + debug.show("e") + return None + if new_name != name: + log( + f"Conflict encountered saving {name} - results stored in {new_name} instead." + ) + else: + try: + with store.open(name) as f: + f.write(content) + except Exception as e: + # Log and then swallow the exception while we're learning. + # Don't let failure pass so quietly when these are the autoritative bits. + log(f"Failed to save {kind}:{name}", e) + return None + raise NotImplementedError() + + +def retrieve_bytes(kind: str, name: str) -> bytes: + store = _get_storage(kind) + with store.open(name) as f: + content = f.read() + return content + +def store_str(kind: str, name: str, content: str, allow_overwrite: bool = False) -> None: + content_bytes = content.encode("utf-8") + store_bytes(kind, name, content_bytes, allow_overwrite) + +def retrieve_str(kind: str, name: str) -> str: + content_bytes = retrieve_bytes(kind, name) + return content_bytes.decode("utf-8") From c8bf16cdb3c92275356df1ead639ee49b7ca2c71 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 11:00:17 -0600 Subject: [PATCH 07/87] feat: test buckets --- docker/scripts/app-configure-blobstore.py | 34 +++------------------- docker/scripts/app-init.sh | 2 +- ietf/doc/storage_utils.py | 6 ++-- ietf/settings.py | 8 ++++++ ietf/settings_test.py | 15 +++------- ietf/submit/tests.py | 2 +- ietf/submit/utils.py | 2 +- ietf/utils/test_runner.py | 35 +++++++++++++++++++++++ requirements.txt | 2 +- 9 files changed, 58 insertions(+), 48 deletions(-) diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index 40202f6fbc..c7e9c71a7f 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -4,6 +4,8 @@ import boto3 import sys +from ietf.settings import MORE_STORAGE_NAMES + def init_blobstore(): blobstore = boto3.resource("s3", endpoint_url="http://blobstore:9000", @@ -13,38 +15,10 @@ def init_blobstore(): config=boto3.session.Config(signature_version="s3v4"), verify=False ) - for bucketname in [ - "agenda", - "bluesheets", - "bofreq", - "charter", - "chatlog", - "conflrev", - "draft-xml", - "draft-txt", - "draft-html", - "draft-htmlized", - "draft-pdf", - "draft-pdfized", - "liai-att", - "liaison", - "minutes", - "narrativeminutes", - "polls", - "procmaterials", - "recording", - "review", - "rfc-txt", - "rfc-html", - "rfc-pdf", - "rfc-htmlized", - "rfc-pdfized", - "slides", - "statchg", - "statement", - ]: + for bucketname in MORE_STORAGE_NAMES: blobstore.create_bucket(Bucket=bucketname) + if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index a3e405560f..e970398ac2 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -76,7 +76,7 @@ chmod +x ./docker/scripts/app-create-dirs.sh # Configure the development blobstore echo "Configuring blobstore..." -python ./docker/scripts/app-configure-blobstore.py +PYTHONPATH=/workspace python ./docker/scripts/app-configure-blobstore.py # Download latest coverage results file diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 9cf336428e..ae36fa34d2 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -12,9 +12,9 @@ def _get_storage(kind: str) -> Storage: "bofreq", "charter", "conflrev", - "draft-xml", - "draft-txt", - "draft-html", + "draft", + "draft", + "draft", ]: return storages[kind] else: diff --git a/ietf/settings.py b/ietf/settings.py index 8819baa03d..57a29ef4c7 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -741,6 +741,14 @@ def skip_unreadable_post(record): "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}, } +# settings_local will need to configure storages for these names +MORE_STORAGE_NAMES = [ + "bofreq", + "charter", + "conflrev", + "draft", +] + # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 20fd9d8993..a71dffb46a 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import boto3, STORAGES, TEST_CODE_COVERAGE_CHECKER +from ietf.settings import boto3, STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES import debug # pyflakes:ignore debug.debug = True @@ -106,15 +106,8 @@ def tempdir_with_cleanup(**kwargs): }, } -for bucketname in [ - "bofreq", - "charter", - "conflrev", - "draft-xml", - "draft-txt", - "draft-html", -]: - STORAGES[bucketname] = { +for storagename in MORE_STORAGE_NAMES: + STORAGES[storagename] = { "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": dict( endpoint_url="http://blobstore:9000", @@ -123,6 +116,6 @@ def tempdir_with_cleanup(**kwargs): security_token=None, client_config=boto3.session.Config(signature_version="s3v4"), verify=False, - bucket_name=bucketname, + bucket_name=f"test-{storagename}", ), } diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 5ad4e3b4f0..9a79a457db 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -429,7 +429,7 @@ def submit_new_wg(self, formats): self.assertTrue(draft.latest_event(type="added_suggested_replaces")) self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))) self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev)))) - self.assertTrue(len(retrieve_str("draft-txt",f"{name}-{rev}.txt"))>0) + self.assertTrue(len(retrieve_str("draft",f"txt/{name}-{rev}.txt"))>0) self.assertEqual(draft.type_id, "draft") self.assertEqual(draft.stream_id, "ietf") self.assertTrue(draft.expires >= timezone.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1)) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 1fb9d4f6b4..f37ceb601a 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -668,7 +668,7 @@ def move_files_to_repository(submission): os.link(dest, ftp_dest) with open(dest,"rb") as f: content_bytes = f.read() - store_bytes(f"draft-{ext}", fname, content_bytes) + store_bytes(f"draft", f"{ext}/{fname}", content_bytes) elif dest.exists(): log.log("Intended to move '%s' to '%s', but found source missing while destination exists.") elif f".{ext}" in submission.file_types.split(','): diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 49d53e1e1d..acd6e5da0c 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -48,6 +48,8 @@ import subprocess import tempfile import copy +import boto3 +import botocore import factory.random import urllib3 import warnings @@ -752,6 +754,7 @@ def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_versio # contains parent classes to later subclasses, the parent classes will determine the ordering, so use the most # specific classes necessary to get the right ordering: self.reorder_by = (PyFlakesTestCase, MyPyTest,) + self.reorder_by + (StaticLiveServerTestCase, TemplateTagTest, CoverageTest,) + self.buckets = set() def setup_test_environment(self, **kwargs): global template_coverage_collection @@ -936,6 +939,27 @@ def setup_test_environment(self, **kwargs): print(" (extra pedantically)") self.vnu = start_vnu_server() + blobstore = boto3.resource("s3", + endpoint_url="http://blobstore:9000", + aws_access_key_id="minio_root", + aws_secret_access_key="minio_pass", + aws_session_token=None, + config=boto3.session.Config(signature_version="s3v4"), + #config=boto3.session.Config(signature_version=botocore.UNSIGNED), + verify=False + ) + for storagename in settings.MORE_STORAGE_NAMES: + bucketname = f"test-{storagename}" + try: + bucket = blobstore.create_bucket(Bucket=bucketname) + #debug.show('f"created {bucket}"') + self.buckets.add(bucket) + except blobstore.meta.client.exceptions.BucketAlreadyOwnedByYou as e: + #debug.show('f"{bucketname} already there: {e}"') + bucket = blobstore.Bucket(bucketname) + self.buckets.add(bucket) + + super(IetfTestRunner, self).setup_test_environment(**kwargs) def teardown_test_environment(self, **kwargs): @@ -966,6 +990,17 @@ def teardown_test_environment(self, **kwargs): if self.vnu: self.vnu.terminate() + + for bucket in self.buckets: + # bucketname=bucket.name + # debug.show('f"Trying to delete {bucketname} contents"') + # debug.show("bucket.objects.delete()") + # debug.show('f"Trying to delete {bucketname} itself"') + # debug.show("bucket.delete()") + bucket.objects.delete() + bucket.delete() + + super(IetfTestRunner, self).teardown_test_environment(**kwargs) def validate(self, testcase): diff --git a/requirements.txt b/requirements.txt index 1431ea1441..0694bf83d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ beautifulsoup4>=4.11.1 # Only used in tests bibtexparser>=1.2.0 # Only used in tests bleach>=6 types-bleach>=6 -boto3>=1.35 +boto3>=1.35,<1.36 celery>=5.2.6 coverage>=4.5.4,<5.0 # Coverage 5.x moves from a json database to SQLite. Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views defusedxml>=0.7.1 # for TastyPie when using xml; not a declared dependency From 6e877cecd5a74a600d3ed057ae637c9c974dada6 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 11:03:18 -0600 Subject: [PATCH 08/87] chore: black --- docker/scripts/app-configure-blobstore.py | 7 ++++--- ietf/doc/storage_utils.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index c7e9c71a7f..b8908a8b3b 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -6,19 +6,20 @@ from ietf.settings import MORE_STORAGE_NAMES + def init_blobstore(): - blobstore = boto3.resource("s3", + blobstore = boto3.resource( + "s3", endpoint_url="http://blobstore:9000", aws_access_key_id="minio_root", aws_secret_access_key="minio_pass", aws_session_token=None, config=boto3.session.Config(signature_version="s3v4"), - verify=False + verify=False, ) for bucketname in MORE_STORAGE_NAMES: blobstore.create_bucket(Bucket=bucketname) - if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index ae36fa34d2..e20e41358d 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -1,12 +1,13 @@ # Copyright The IETF Trust 2025, All Rights Reserved -import debug # pyflakes ignore +import debug # pyflakes ignore from django.core.files.base import ContentFile from django.core.files.storage import storages, Storage from ietf.utils.log import log + def _get_storage(kind: str) -> Storage: if kind in [ "bofreq", @@ -22,7 +23,9 @@ def _get_storage(kind: str) -> Storage: raise NotImplementedError(f"Don't know how to store {kind}") -def store_bytes(kind: str, name: str, content: bytes, allow_overwrite: bool = False) -> None: +def store_bytes( + kind: str, name: str, content: bytes, allow_overwrite: bool = False +) -> None: store = _get_storage(kind) if not allow_overwrite: try: @@ -55,10 +58,14 @@ def retrieve_bytes(kind: str, name: str) -> bytes: content = f.read() return content -def store_str(kind: str, name: str, content: str, allow_overwrite: bool = False) -> None: + +def store_str( + kind: str, name: str, content: str, allow_overwrite: bool = False +) -> None: content_bytes = content.encode("utf-8") store_bytes(kind, name, content_bytes, allow_overwrite) + def retrieve_str(kind: str, name: str) -> str: content_bytes = retrieve_bytes(kind, name) return content_bytes.decode("utf-8") From cdc29189319111b5c8b251243f5b55fef41cdcf4 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 11:51:24 -0600 Subject: [PATCH 09/87] chore: remove unused import --- ietf/utils/test_runner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index acd6e5da0c..3412fc633c 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -49,7 +49,6 @@ import tempfile import copy import boto3 -import botocore import factory.random import urllib3 import warnings @@ -954,8 +953,8 @@ def setup_test_environment(self, **kwargs): bucket = blobstore.create_bucket(Bucket=bucketname) #debug.show('f"created {bucket}"') self.buckets.add(bucket) - except blobstore.meta.client.exceptions.BucketAlreadyOwnedByYou as e: - #debug.show('f"{bucketname} already there: {e}"') + except blobstore.meta.client.exceptions.BucketAlreadyOwnedByYou: + #debug.show('f"{bucketname} already there"') bucket = blobstore.Bucket(bucketname) self.buckets.add(bucket) From 6baf663e47001e2a35d0ac676bdae2a4088b6e69 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 11:52:16 -0600 Subject: [PATCH 10/87] chore: avoid f string when not needed --- ietf/submit/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index f37ceb601a..9f9306e111 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -668,7 +668,7 @@ def move_files_to_repository(submission): os.link(dest, ftp_dest) with open(dest,"rb") as f: content_bytes = f.read() - store_bytes(f"draft", f"{ext}/{fname}", content_bytes) + store_bytes("draft", f"{ext}/{fname}", content_bytes) elif dest.exists(): log.log("Intended to move '%s' to '%s', but found source missing while destination exists.") elif f".{ext}" in submission.file_types.split(','): From d79abbd21eed6619ceda26c46c7c23b96e5ae7a1 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 11:53:02 -0600 Subject: [PATCH 11/87] fix: inform all settings files about blobstores --- dev/deploy-to-container/settings_local.py | 14 +++++++++++++ dev/diff/settings_local.py | 14 +++++++++++++ dev/tests/settings_local.py | 14 +++++++++++++ docker/configs/settings_local.py | 25 ++++++++++++----------- 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 0a991ae9fe..4f6b5243b1 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -79,3 +79,17 @@ # OIDC configuration SITE_URL = 'https://__HOSTNAME__' + +for storagename in MORE_STORAGE_NAMES: + STORAGES[storagename] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url="http://blobstore:9000", + access_key="minio_root", + secret_key="minio_pass", + security_token=None, + client_config=boto3.session.Config(signature_version="s3v4"), + verify=False, + bucket_name=f"test-{storagename}", + ), + } diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index 95d1e481c9..ada1c7a568 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -66,3 +66,17 @@ SLIDE_STAGING_PATH = 'test/staging/' DE_GFM_BINARY = '/usr/local/bin/de-gfm' + +for storagename in MORE_STORAGE_NAMES: + STORAGES[storagename] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url="http://blobstore:9000", + access_key="minio_root", + secret_key="minio_pass", + security_token=None, + client_config=boto3.session.Config(signature_version="s3v4"), + verify=False, + bucket_name=f"test-{storagename}", + ), + } diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index 7b10bee06a..9218a6e036 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -65,3 +65,17 @@ SLIDE_STAGING_PATH = 'test/staging/' DE_GFM_BINARY = '/usr/local/bin/de-gfm' + +for storagename in MORE_STORAGE_NAMES: + STORAGES[storagename] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url="http://blobstore:9000", + access_key="minio_root", + secret_key="minio_pass", + security_token=None, + client_config=boto3.session.Config(signature_version="s3v4"), + verify=False, + bucket_name=f"test-{storagename}", + ), + } diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 1b458324e1..f77d427ea3 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -38,18 +38,19 @@ # DEV_TEMPLATE_CONTEXT_PROCESSORS = [ # 'ietf.context_processors.sql_debug', # ] -STORAGES["ietfdata"] = { - "BACKEND": "storages.backends.s3.S3Storage", - "OPTIONS": dict( - endpoint_url="http://blobstore:9000", - access_key="minio_root", - secret_key="minio_pass", - security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), - verify=False, - bucket_name="ietfdata", - ), -} +for storagename in MORE_STORAGE_NAMES: + STORAGES[storagename] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url="http://blobstore:9000", + access_key="minio_root", + secret_key="minio_pass", + security_token=None, + client_config=boto3.session.Config(signature_version="s3v4"), + verify=False, + bucket_name=storagename, + ), + } DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/' From a46165bcdb1b308169d3a20aa4a3515dc3107912 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 14:28:59 -0600 Subject: [PATCH 12/87] fix: declare types for some settings --- ietf/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 57a29ef4c7..5f24c5f14a 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -736,13 +736,13 @@ def skip_unreadable_post(record): "schedule_name": r"(?P[A-Za-z0-9-:_]+)", } -STORAGES = { +STORAGES: dict[str, Any] = { "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}, } # settings_local will need to configure storages for these names -MORE_STORAGE_NAMES = [ +MORE_STORAGE_NAMES: list[str] = [ "bofreq", "charter", "conflrev", From 2304db869090bc69625b3e5e2ca05bc3d30b54c2 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 15:06:21 -0600 Subject: [PATCH 13/87] ci: point to new target base --- dev/build/TARGET_BASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index b5d33714f2..fd2d539a9e 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20241212T1741 +20250116T2033 From 13b828e2d1357d96703f3027d79bba8870fcdc09 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 15:32:01 -0600 Subject: [PATCH 14/87] ci: adjust test workflow --- .github/workflows/tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5457415f59..0198d60bc3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,12 @@ jobs: services: db: image: ghcr.io/ietf-tools/datatracker-db:latest + blobstore: + image: quay.io/minio/minio:latest + environment: + - MINIO_ROOT_USER=minio_root + - MINIO_ROOT_PASSWORD=minio_pass + - MINIO_DEFAULT_BUCKETS=defaultbucket steps: - uses: actions/checkout@v4 From 7223a9c8643b231429aa7aa77e484250dfe3f79e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 16:56:38 -0600 Subject: [PATCH 15/87] fix: give the tests debug environment a blobstore --- dev/tests/docker-compose.debug.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml index 8d939e0ea2..95e497789b 100644 --- a/dev/tests/docker-compose.debug.yml +++ b/dev/tests/docker-compose.debug.yml @@ -28,5 +28,13 @@ services: volumes: - postgresdb-data:/var/lib/postgresql/data + blobstore: + image: quay.io/minio/minio:latest + environment: + - MINIO_ROOT_USER=minio_root + - MINIO_ROOT_PASSWORD=minio_pass + - MINIO_DEFAULT_BUCKETS=defaultbucket + command: server /data + volumes: postgresdb-data: From 5ad71a5af54a0b4fb3559c2c5b8d046e5eb51df3 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Jan 2025 16:57:02 -0600 Subject: [PATCH 16/87] fix: "better" name declarations --- dev/deploy-to-container/settings_local.py | 3 ++- dev/diff/settings_local.py | 3 ++- dev/tests/settings_local.py | 3 ++- docker/configs/settings_local.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 4f6b5243b1..2908a1d97c 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -1,7 +1,8 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- -from ietf.settings import * # pyflakes:ignore +from ietf.settings import * # pyflakes:ignore +from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES ALLOWED_HOSTS = ['*'] diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index ada1c7a568..785c6b5648 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -1,7 +1,8 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- -from ietf.settings import * # pyflakes:ignore +from ietf.settings import * # pyflakes:ignore +from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES ALLOWED_HOSTS = ['*'] diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index 9218a6e036..42ab6a019a 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -1,7 +1,8 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- -from ietf.settings import * # pyflakes:ignore +from ietf.settings import * # pyflakes:ignore +from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES ALLOWED_HOSTS = ['*'] diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index f77d427ea3..850cdd017d 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -import boto3 +from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES ALLOWED_HOSTS = ['*'] From 44a49ff35649ee0b575bd1ce071ce22cdb259089 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 17 Jan 2025 13:20:15 -0600 Subject: [PATCH 17/87] ci: use devblobstore container --- .github/workflows/tests.yml | 6 +----- dev/tests/docker-compose.debug.yml | 7 +------ docker-compose.yml | 7 +------ 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0198d60bc3..f10c1db9a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,11 +29,7 @@ jobs: db: image: ghcr.io/ietf-tools/datatracker-db:latest blobstore: - image: quay.io/minio/minio:latest - environment: - - MINIO_ROOT_USER=minio_root - - MINIO_ROOT_PASSWORD=minio_pass - - MINIO_DEFAULT_BUCKETS=defaultbucket + image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - uses: actions/checkout@v4 diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml index 95e497789b..8117b92375 100644 --- a/dev/tests/docker-compose.debug.yml +++ b/dev/tests/docker-compose.debug.yml @@ -29,12 +29,7 @@ services: - postgresdb-data:/var/lib/postgresql/data blobstore: - image: quay.io/minio/minio:latest - environment: - - MINIO_ROOT_USER=minio_root - - MINIO_ROOT_PASSWORD=minio_pass - - MINIO_DEFAULT_BUCKETS=defaultbucket - command: server /data + image: ghcr.io/ietf-tools/datatracker-devblobstore:latest volumes: postgresdb-data: diff --git a/docker-compose.yml b/docker-compose.yml index 89faee5ebf..ce2e42269a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,15 +85,10 @@ services: - app-assets:/assets blobstore: - image: quay.io/minio/minio:latest + image: ghcr.io/ietf-tools/datatracker-devblobstore:latest restart: unless-stopped volumes: - "minio-data:/data" - environment: - - MINIO_ROOT_USER=minio_root - - MINIO_ROOT_PASSWORD=minio_pass - - MINIO_DEFAULT_BUCKETS=defaultbucket - command: server --console-address ":9001" /data From 9d6b1211a360127d2f347b268df7b52cd2cf8d42 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 17 Jan 2025 15:22:39 -0600 Subject: [PATCH 18/87] chore: identify places to write to blobstorage --- ietf/doc/views_material.py | 1 + ietf/doc/views_statement.py | 4 ++++ ietf/doc/views_status_change.py | 1 + ietf/idindex/tasks.py | 2 ++ ietf/ipr/utils.py | 3 ++- ietf/liaisons/forms.py | 1 + ietf/meeting/forms.py | 1 + ietf/meeting/helpers.py | 5 +++++ ietf/meeting/utils.py | 4 ++++ ietf/meeting/views.py | 2 ++ ietf/submit/utils.py | 4 ++++ 11 files changed, 27 insertions(+), 1 deletion(-) diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index 361bf5f1e2..cd97b59e9b 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -167,6 +167,7 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): with filepath.open('wb+') as dest: for chunk in f.chunks(): dest.write(chunk) + # TODO-BLOBSTORE store (in chunks? is ContentFile good enough?) if not doc.meeting_related(): log.assertion('doc.type_id == "slides"') ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename diff --git a/ietf/doc/views_statement.py b/ietf/doc/views_statement.py index bf9f47ddfe..e95c92e569 100644 --- a/ietf/doc/views_statement.py +++ b/ietf/doc/views_statement.py @@ -139,8 +139,10 @@ def submit(request, name): if writing_pdf: for chunk in form.cleaned_data["statement_file"].chunks(): destination.write(chunk) + # TODO-BLOBSTORE else: destination.write(markdown_content) + # TODO-BLOBSTORE return redirect("ietf.doc.views_doc.document_main", name=statement.name) else: @@ -256,8 +258,10 @@ def new_statement(request): if writing_pdf: for chunk in form.cleaned_data["statement_file"].chunks(): destination.write(chunk) + # TODO-BLOBSTORE else: destination.write(markdown_content) + # TODO-BLOBSTORE return redirect("ietf.doc.views_doc.document_main", name=statement.name) else: diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index 33b822348a..0f64e2af44 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -163,6 +163,7 @@ def save(self, doc): destination.write(self.cleaned_data['txt']) else: destination.write(self.cleaned_data['content']) + # TODO-BLOBSTORE try: ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10 diff --git a/ietf/idindex/tasks.py b/ietf/idindex/tasks.py index 5e7e193bba..2abec2bf5e 100644 --- a/ietf/idindex/tasks.py +++ b/ietf/idindex/tasks.py @@ -39,6 +39,8 @@ def move_into_place(self, src_path: Path, dest_path: Path, hardlink_dirs: List[P target.unlink(missing_ok=True) os.link(dest_path, target) # until python>=3.10 + # TODO-BLOBSTORE : Going to need something to put these generated things into storage + def cleanup(self): for tf_path in self.cleanup_list: tf_path.unlink(missing_ok=True) diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py index 8f0b9cf3f2..ca6dd85f66 100644 --- a/ietf/ipr/utils.py +++ b/ietf/ipr/utils.py @@ -87,8 +87,9 @@ def generate_draft_recursive_txt(): filename = '/a/ietfdata/derived/ipr_draft_recursive.txt' with open(filename, 'w') as f: f.write(data) + # TODO-BLOBSTORE: Where is this exposed? + - def ingest_response_email(message: bytes): from ietf.api.views import EmailIngestionError # avoid circular import try: diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 1d91041b25..8e6afa3a4f 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -379,6 +379,7 @@ def save_attachments(self): attach_file = io.open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb') attach_file.write(attached_file.read()) attach_file.close() + # TODO-BLOBSTORE if not self.is_new: # create modified event diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index 3b66d2cd29..743e33d438 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -361,6 +361,7 @@ def save_agenda(self): os.makedirs(directory) with io.open(path, "w", encoding='utf-8') as file: file.write(self.cleaned_data['agenda']) + # TODO-BLOBSTORE class InterimAnnounceForm(forms.ModelForm): diff --git a/ietf/meeting/helpers.py b/ietf/meeting/helpers.py index 7f1c85990e..39d271ae6b 100644 --- a/ietf/meeting/helpers.py +++ b/ietf/meeting/helpers.py @@ -649,6 +649,11 @@ def read_session_file(type, num, doc): def read_agenda_file(num, doc): return read_session_file('agenda', num, doc) +# TODO-BLOBSTORE: this is _yet another_ draft derived variant created when users +# ask for drafts from the meeting agenda page. Consider whether to refactor this +# now to not call out to external binaries, and consider whether we need this extra +# format at all in the draft blobstore. if so, it would probably be stored under +# something like plainpdf/ def convert_draft_to_pdf(doc_name): inpath = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, doc_name + ".txt") outpath = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf") diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index b68a311f5d..c760d97313 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -229,6 +229,7 @@ def generate_bluesheet(request, session): os.close(fd) with open(name, "w") as file: file.write(text) + # TODO-BLOBSTORE with open(name, "br") as file: return save_bluesheet(request, session, file) @@ -788,6 +789,7 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N # document (sanitize will remove these). clean = sanitize_document(text) destination.write(clean.encode('utf8')) + # TODO-BLOBSTORE if request and clean != text: messages.warning(request, ( @@ -798,6 +800,7 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N else: for chunk in chunks: destination.write(chunk) + # TODO-BLOBSTORE # unzip zipfile if is_zipfile: @@ -834,6 +837,7 @@ def write_doc_for_session(session, type_id, filename, contents): path.mkdir(parents=True, exist_ok=True) with open(path / filename, "wb") as file: file.write(contents.encode('utf-8')) + # TODO-BLOBSTORE return def create_recording(session, url, title=None, user=None): diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index f386f8932f..0498876f35 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -3001,6 +3001,7 @@ def upload_session_slides(request, session_id, num, name=None): for chunk in file.chunks(): destination.write(chunk) destination.close() + # TODO-BLOBSTORE : Do we keep a staging blobstore until we can refactor away exposing staged things through www.ietf.org? submission.filename = filename submission.save() @@ -4703,6 +4704,7 @@ def err(code, text): save_err = save_bluesheet(request, session, file) if save_err: return err(400, save_err) + # TODO-BLOBSTORE return HttpResponse("Done", status=200, content_type='text/plain') diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 9063b93406..2a050bb7dd 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -499,6 +499,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev )) with io.open(ref_rev_file_name, "w", encoding='utf-8') as f: f.write(ref_text) + # TODO-BLOBSTORE log.log(f"{submission.name}: done") @@ -773,6 +774,7 @@ def save_files(form): for chunk in f.chunks(): destination.write(chunk) log.log("saved file %s" % name) + # TODO-BLOBSTORE return file_name @@ -995,6 +997,7 @@ def render_missing_formats(submission): xml_version, ) ) + # TODO-BLOBSTORE # --- Convert to html --- html_path = staging_path(submission.name, submission.rev, '.html') @@ -1017,6 +1020,7 @@ def render_missing_formats(submission): xml_version, ) ) + # TODO-BLOBSTORE def accept_submission(submission: Submission, request: Optional[HttpRequest] = None, autopost=False): From 9092b1ba3e9df270cd8effdbb0ec2a2bd2a516d7 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 24 Jan 2025 10:31:36 -0600 Subject: [PATCH 19/87] chore: remove unreachable code --- ietf/meeting/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index b17c84f313..cfe7adfae7 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -744,14 +744,6 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N path = Path(meeting.get_materials_path()) / subdir path.mkdir(parents=True, exist_ok=True) - # agendas and minutes can only have one file instance so delete file if it already exists - if subdir in ('agenda', 'minutes'): - for f in path.glob(f'{filename.stem}.*'): - try: - f.unlink() - except FileNotFoundError: - pass # if the file is already gone, so be it - with (path / filename).open('wb+') as destination: # prep file for reading if hasattr(file, "chunks"): From fa863af2d0d822b73f353c76b579f04c6adcbfc5 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 24 Jan 2025 15:09:31 -0600 Subject: [PATCH 20/87] feat: store materials --- ietf/doc/storage_utils.py | 54 +++++++++++++++------------------ ietf/doc/tests_material.py | 6 ++++ ietf/doc/views_material.py | 4 ++- ietf/meeting/models.py | 1 + ietf/meeting/tests_views.py | 59 ++++++++++++++++++++++++++++++------- ietf/meeting/utils.py | 12 ++++++-- ietf/settings.py | 6 ++++ 7 files changed, 97 insertions(+), 45 deletions(-) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index e20e41358d..022dfa1136 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -2,34 +2,29 @@ import debug # pyflakes ignore -from django.core.files.base import ContentFile +from django.conf import settings +from django.core.files.base import ContentFile, File from django.core.files.storage import storages, Storage from ietf.utils.log import log def _get_storage(kind: str) -> Storage: - if kind in [ - "bofreq", - "charter", - "conflrev", - "draft", - "draft", - "draft", - ]: + if kind in settings.MORE_STORAGE_NAMES: return storages[kind] else: debug.say(f"Got into not-implemented looking for {kind}") raise NotImplementedError(f"Don't know how to store {kind}") -def store_bytes( - kind: str, name: str, content: bytes, allow_overwrite: bool = False -) -> None: +def store_file(kind: str, name: str, file: File, allow_overwrite: bool = False) -> None: store = _get_storage(kind) - if not allow_overwrite: + if not allow_overwrite and store.exists(name): + log(f"Failed to save {kind}:{name} - name already exists in store") + debug.show('f"Failed to save {kind}:{name} - name already exists in store"') + else: try: - new_name = store.save(name, ContentFile(content)) + new_name = store.save(name, file) except Exception as e: # Log and then swallow the exception while we're learning. # Don't let failure pass so quietly when these are the autoritative bits. @@ -40,32 +35,31 @@ def store_bytes( log( f"Conflict encountered saving {name} - results stored in {new_name} instead." ) - else: - try: - with store.open(name) as f: - f.write(content) - except Exception as e: - # Log and then swallow the exception while we're learning. - # Don't let failure pass so quietly when these are the autoritative bits. - log(f"Failed to save {kind}:{name}", e) - return None - raise NotImplementedError() + debug.show('f"Conflict encountered saving {name} - results stored in {new_name} instead."') + return None -def retrieve_bytes(kind: str, name: str) -> bytes: - store = _get_storage(kind) - with store.open(name) as f: - content = f.read() - return content +def store_bytes( + kind: str, name: str, content: bytes, allow_overwrite: bool = False +) -> None: + return store_file(kind, name, ContentFile(content), allow_overwrite) def store_str( kind: str, name: str, content: str, allow_overwrite: bool = False ) -> None: content_bytes = content.encode("utf-8") - store_bytes(kind, name, content_bytes, allow_overwrite) + return store_bytes(kind, name, content_bytes, allow_overwrite) + + +def retrieve_bytes(kind: str, name: str) -> bytes: + store = _get_storage(kind) + with store.open(name) as f: + content = f.read() + return content def retrieve_str(kind: str, name: str) -> str: content_bytes = retrieve_bytes(kind, name) + # TODO: try to decode all the different ways doc.text() does return content_bytes.decode("utf-8") diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py index aaea8fec3d..c87341c95b 100644 --- a/ietf/doc/tests_material.py +++ b/ietf/doc/tests_material.py @@ -18,6 +18,7 @@ from django.utils import timezone from ietf.doc.models import Document, State, NewRevisionDocEvent +from ietf.doc.storage_utils import retrieve_str from ietf.group.factories import RoleFactory from ietf.group.models import Group from ietf.meeting.factories import MeetingFactory, SessionFactory, SessionPresentationFactory @@ -123,6 +124,9 @@ def test_upload_slides(self): ftp_filepath=Path(settings.FTP_DIR) / "slides" / basename with ftp_filepath.open() as f: self.assertEqual(f.read(), content) + # This test is very sloppy wrt the actual file content. + # Working with/around that for the moment. + self.assertEqual(retrieve_str("slides", basename), content) # check that posting same name is prevented test_file.seek(0) @@ -237,4 +241,6 @@ def test_revise(self, mock_slides_manager_cls): with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f: self.assertEqual(f.read(), content) + self.assertEqual(retrieve_str("slides", f"{doc.name}-{doc.rev}.txt"), content) + diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index cd97b59e9b..afc833eb95 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -19,6 +19,7 @@ from ietf.doc.models import Document, DocTypeName, DocEvent, State from ietf.doc.models import NewRevisionDocEvent +from ietf.doc.storage_utils import store_file from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules from ietf.group.models import Group from ietf.group.utils import can_manage_materials @@ -167,7 +168,8 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): with filepath.open('wb+') as dest: for chunk in f.chunks(): dest.write(chunk) - # TODO-BLOBSTORE store (in chunks? is ContentFile good enough?) + f.seek(0) + store_file(doc.type_id, basename, f) if not doc.meeting_related(): log.assertion('doc.type_id == "slides"') ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 8c6fb97413..ecc620cb65 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1431,6 +1431,7 @@ class MeetingHost(models.Model): """Meeting sponsor""" meeting = ForeignKey(Meeting, related_name='meetinghosts') name = models.CharField(max_length=255, blank=False) + # TODO-BLOBSTORE - capture these logos logo = MissingOkImageField( storage=NoLocationMigrationFileSystemStorage(location=settings.MEETINGHOST_LOGO_PATH), upload_to=_host_upload_path, diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 0647da52ab..5ba18cf714 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -37,6 +37,7 @@ import debug # pyflakes:ignore from ietf.doc.models import Document, NewRevisionDocEvent +from ietf.doc.storage_utils import retrieve_bytes, retrieve_str from ietf.group.models import Group, Role, GroupFeatures from ietf.group.utils import can_manage_group from ietf.person.models import Person @@ -6310,12 +6311,14 @@ def test_upload_bluesheets(self): q = PyQuery(r.content) self.assertIn('Upload', str(q("title"))) self.assertFalse(session.presentations.exists()) - test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test') + test_content = '%PDF-1.4\n%âãÏÓ\nthis is some text for a test' + test_file = StringIO(test_content) test_file.name = "not_really.pdf" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 302) bs_doc = session.presentations.filter(document__type_id='bluesheets').first().document self.assertEqual(bs_doc.rev,'00') + self.assertEqual(retrieve_str("bluesheets", f"{bs_doc.name}-{bs_doc.rev}.pdf"), test_content) r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -6345,12 +6348,14 @@ def test_upload_bluesheets_interim(self): q = PyQuery(r.content) self.assertIn('Upload', str(q("title"))) self.assertFalse(session.presentations.exists()) - test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test') + test_content = '%PDF-1.4\n%âãÏÓ\nthis is some text for a test' + test_file = StringIO(test_content) test_file.name = "not_really.pdf" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 302) bs_doc = session.presentations.filter(document__type_id='bluesheets').first().document self.assertEqual(bs_doc.rev,'00') + self.assertEqual(retrieve_str("bluesheets", f"{bs_doc.name}-{bs_doc.rev}.pdf"), test_content) def test_upload_bluesheets_interim_chair_access(self): make_meeting_test_data() @@ -6424,27 +6429,37 @@ def test_upload_minutes_agenda(self): self.assertIn('Some text', text) self.assertNotIn('
', text) self.assertIn('charset="utf-8"', text) + text = retrieve_str(doctype, f"{doc.name}-{doc.rev}.html") + self.assertIn('Some text', text) + self.assertNotIn('
', text) + self.assertIn('charset="utf-8"', text) # txt upload - test_file = BytesIO(b'This is some text for a test, with the word\nvirtual at the beginning of a line.') + test_bytes = b'This is some text for a test, with the word\nvirtual at the beginning of a line.' + test_file = BytesIO(test_bytes) test_file.name = "some.txt" r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=False)) self.assertEqual(r.status_code, 302) doc = session.presentations.filter(document__type_id=doctype).first().document self.assertEqual(doc.rev,'01') self.assertFalse(session2.presentations.filter(document__type_id=doctype)) + retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt") + self.assertEqual(retrieved_bytes, test_bytes) r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertIn('Revise', str(q("Title"))) - test_file = BytesIO(b'this is some different text for a test') + test_bytes = b'this is some different text for a test' + test_file = BytesIO(test_bytes) test_file.name = "also_some.txt" r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=True)) self.assertEqual(r.status_code, 302) doc = Document.objects.get(pk=doc.pk) self.assertEqual(doc.rev,'02') self.assertTrue(session2.presentations.filter(document__type_id=doctype)) + retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt") + self.assertEqual(retrieved_bytes, test_bytes) # Test bad encoding test_file = BytesIO('

Title

Some\x93text
'.encode('latin1')) @@ -6497,12 +6512,15 @@ def test_upload_minutes_agenda_interim(self): q = PyQuery(r.content) self.assertIn('Upload', str(q("title"))) self.assertFalse(session.presentations.filter(document__type_id=doctype)) - test_file = BytesIO(b'this is some text for a test') + test_bytes = b'this is some text for a test' + test_file = BytesIO(test_bytes) test_file.name = "not_really.txt" r = self.client.post(url,dict(submission_method="upload",file=test_file)) self.assertEqual(r.status_code, 302) doc = session.presentations.filter(document__type_id=doctype).first().document self.assertEqual(doc.rev,'00') + retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt") + self.assertEqual(retrieved_bytes, test_bytes) # Verify that we don't have dead links url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym}) @@ -6524,12 +6542,15 @@ def test_upload_narrativeminutes(self): q = PyQuery(r.content) self.assertIn('Upload', str(q("title"))) self.assertFalse(session.presentations.filter(document__type_id=doctype)) - test_file = BytesIO(b'this is some text for a test') + test_bytes = b'this is some text for a test' + test_file = BytesIO(test_bytes) test_file.name = "not_really.txt" r = self.client.post(url,dict(submission_method="upload",file=test_file)) self.assertEqual(r.status_code, 302) doc = session.presentations.filter(document__type_id=doctype).first().document self.assertEqual(doc.rev,'00') + retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt") + self.assertEqual(retrieved_bytes, test_bytes) # Verify that we don't have dead links url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym}) @@ -6554,18 +6575,22 @@ def test_enter_agenda(self): self.assertRedirects(r, redirect_url) doc = session.presentations.filter(document__type_id='agenda').first().document self.assertEqual(doc.rev,'00') + self.assertEqual(retrieve_str("agenda",f"{doc.name}-{doc.rev}.md"), test_text) r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertIn('Revise', str(q("Title"))) - test_file = BytesIO(b'Upload after enter') + test_bytes = b'Upload after enter' + test_file = BytesIO(test_bytes) test_file.name = "some.txt" r = self.client.post(url,dict(submission_method="upload",file=test_file)) self.assertRedirects(r, redirect_url) doc = Document.objects.get(pk=doc.pk) self.assertEqual(doc.rev,'01') + retrieved_bytes = retrieve_bytes("agenda", f"{doc.name}-{doc.rev}.txt") + self.assertEqual(retrieved_bytes, test_bytes) r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -6577,6 +6602,8 @@ def test_enter_agenda(self): self.assertRedirects(r, redirect_url) doc = Document.objects.get(pk=doc.pk) self.assertEqual(doc.rev,'02') + self.assertEqual(retrieve_str("agenda",f"{doc.name}-{doc.rev}.md"), test_text) + @override_settings(MEETECHO_API_CONFIG="fake settings") # enough to trigger API calls @patch("ietf.meeting.views.SlidesManager") @@ -6592,7 +6619,8 @@ def test_upload_slides(self, mock_slides_manager_cls): q = PyQuery(r.content) self.assertIn('Upload', str(q("title"))) self.assertFalse(session1.presentations.filter(document__type_id='slides')) - test_file = BytesIO(b'this is not really a slide') + test_bytes = b'this is not really a slide' + test_file = BytesIO(test_bytes) test_file.name = 'not_really.txt' r = self.client.post(url,dict(file=test_file,title='a test slide file',apply_to_all=True,approved=True)) self.assertEqual(r.status_code, 302) @@ -6604,6 +6632,7 @@ def test_upload_slides(self, mock_slides_manager_cls): self.assertEqual(mock_slides_manager_cls.call_count, 1) self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings")) self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 2) + self.assertEqual(retrieve_bytes("slides", f"{sp.document.name}-{sp.document.rev}.txt"), test_bytes) # don't care which order they were called in, just that both sessions were updated self.assertCountEqual( mock_slides_manager_cls.return_value.add.call_args_list, @@ -6615,7 +6644,8 @@ def test_upload_slides(self, mock_slides_manager_cls): mock_slides_manager_cls.reset_mock() url = urlreverse('ietf.meeting.views.upload_session_slides',kwargs={'num':session2.meeting.number,'session_id':session2.id}) - test_file = BytesIO(b'some other thing still not slidelike') + test_bytes = b'some other thing still not slidelike' + test_file = BytesIO(test_bytes) test_file.name = 'also_not_really.txt' r = self.client.post(url,dict(file=test_file,title='a different slide file',apply_to_all=False,approved=True)) self.assertEqual(r.status_code, 302) @@ -6628,6 +6658,7 @@ def test_upload_slides(self, mock_slides_manager_cls): self.assertEqual(mock_slides_manager_cls.call_count, 1) self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings")) self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 1) + self.assertEqual(retrieve_bytes("slides", f"{sp.document.name}-{sp.document.rev}.txt"), test_bytes) self.assertEqual( mock_slides_manager_cls.return_value.add.call_args, call(session=session2, slides=sp.document, order=2), @@ -6639,7 +6670,8 @@ def test_upload_slides(self, mock_slides_manager_cls): self.assertTrue(r.status_code, 200) q = PyQuery(r.content) self.assertIn('Revise', str(q("title"))) - test_file = BytesIO(b'new content for the second slide deck') + test_bytes = b'new content for the second slide deck' + test_file = BytesIO(test_bytes) test_file.name = 'doesnotmatter.txt' r = self.client.post(url,dict(file=test_file,title='rename the presentation',apply_to_all=False, approved=True)) self.assertEqual(r.status_code, 302) @@ -6649,6 +6681,7 @@ def test_upload_slides(self, mock_slides_manager_cls): self.assertEqual(replacement_sp.rev,'01') self.assertEqual(replacement_sp.document.rev,'01') self.assertEqual(mock_slides_manager_cls.call_count, 1) + self.assertEqual(retrieve_bytes("slides", f"{replacement_sp.document.name}-{replacement_sp.document.rev}.txt"), test_bytes) self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings")) self.assertEqual(mock_slides_manager_cls.return_value.revise.call_count, 1) self.assertEqual( @@ -6728,7 +6761,7 @@ def test_remove_sessionpresentation(self, mock_slides_manager_cls): self.assertEqual(2, agenda.docevent_set.count()) self.assertFalse(mock_slides_manager_cls.called) - + # TODO-BLOBSTORE - review proposed slides def test_propose_session_slides(self): for type_id in ['ietf','interim']: session = SessionFactory(meeting__type_id=type_id) @@ -7071,6 +7104,10 @@ def test_imports_previewed_text(self): minutes_path = Path(self.meeting.get_materials_path()) / 'minutes' with (minutes_path / self.session.minutes().uploaded_filename).open() as f: self.assertEqual(f.read(), 'original markdown text') + self.assertEqual( + retrieve_str("minutes", self.session.minutes().uploaded_filename), + 'original markdown text' + ) def test_refuses_identical_import(self): """Should not be able to import text identical to the current revision""" diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index e1d57f11ea..5b3badbee9 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -21,6 +21,7 @@ import debug # pyflakes:ignore from ietf.dbtemplate.models import DBTemplate +from ietf.doc.storage_utils import store_bytes from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, Constraint, SchedTimeSessAssignment, SessionPresentation, Attended) from ietf.doc.models import Document, State, NewRevisionDocEvent @@ -775,8 +776,10 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N # Whole file sanitization; add back what's missing from a complete # document (sanitize will remove these). clean = sanitize_document(text) - destination.write(clean.encode('utf8')) - # TODO-BLOBSTORE + clean_bytes = clean.encode('utf8') + destination.write(clean_bytes) + # Assumes contents of subdir are always document type ids + store_bytes(subdir, filename.name, clean_bytes) if request and clean != text: messages.warning(request, ( @@ -787,7 +790,10 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N else: for chunk in chunks: destination.write(chunk) - # TODO-BLOBSTORE + file.seek(0) + if hasattr(file, "chunks"): + chunks = file.chunks() + store_bytes(subdir, filename.name, b"".join(chunks)) return None diff --git a/ietf/settings.py b/ietf/settings.py index de8ba2d5be..04d0afec0b 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -748,6 +748,12 @@ def skip_unreadable_post(record): "charter", "conflrev", "draft", + "slides", + "minutes", + "agenda", + "bluesheets", + "procmaterials", + "narrativeminutes", ] # Override this in settings_local.py if needed From c49293be59c26416f4ff8968b8edc70d8157126b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 24 Jan 2025 16:25:39 -0600 Subject: [PATCH 21/87] feat: store statements --- ietf/doc/tests_statement.py | 17 +++++++++++++++++ ietf/doc/views_statement.py | 18 +++++++++++------- ietf/settings.py | 1 + 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/ietf/doc/tests_statement.py b/ietf/doc/tests_statement.py index 2071018b10..fea42b97d6 100644 --- a/ietf/doc/tests_statement.py +++ b/ietf/doc/tests_statement.py @@ -14,6 +14,7 @@ from ietf.doc.factories import StatementFactory, DocEventFactory from ietf.doc.models import Document, State, NewRevisionDocEvent +from ietf.doc.storage_utils import retrieve_str from ietf.group.models import Group from ietf.person.factories import PersonFactory from ietf.utils.mail import outbox, empty_outbox @@ -185,8 +186,16 @@ def test_submit(self): self.assertEqual("%02d" % (int(rev) + 1), doc.rev) if postdict["statement_submission"] == "enter": self.assertEqual(f"# {username}", doc.text()) + self.assertEqual( + retrieve_str("statement", f"{doc.name}-{doc.rev}.md"), + f"# {username}" + ) else: self.assertEqual("not valid pdf", doc.text()) + self.assertEqual( + retrieve_str("statement", f"{doc.name}-{doc.rev}.pdf"), + "not valid pdf" + ) self.assertEqual(docevent_count + 1, doc.docevent_set.count()) self.assertEqual(0, len(outbox)) rev = doc.rev @@ -255,8 +264,16 @@ def test_start_new_statement(self): self.assertIsNotNone(statement.history_set.last().latest_event(type="published_statement")) if postdict["statement_submission"] == "enter": self.assertEqual(statement.text_or_error(), "some stuff") + self.assertEqual( + retrieve_str("statement", statement.uploaded_filename), + "some stuff" + ) else: self.assertTrue(statement.uploaded_filename.endswith("pdf")) + self.assertEqual( + retrieve_str("statement", f"{statement.name}-{statement.rev}.pdf"), + "not valid pdf" + ) self.assertEqual(len(outbox), 0) existing_statement = StatementFactory() diff --git a/ietf/doc/views_statement.py b/ietf/doc/views_statement.py index e95c92e569..1af70601cc 100644 --- a/ietf/doc/views_statement.py +++ b/ietf/doc/views_statement.py @@ -10,6 +10,7 @@ from django.views.decorators.cache import cache_control from django.shortcuts import get_object_or_404, render, redirect from django.template.loader import render_to_string +from ietf.doc.storage_utils import store_file, store_str from ietf.utils import markdown from django.utils.html import escape @@ -137,14 +138,15 @@ def submit(request, name): mode="wb" if writing_pdf else "w" ) as destination: if writing_pdf: - for chunk in form.cleaned_data["statement_file"].chunks(): + f = form.cleaned_data["statement_file"] + for chunk in f.chunks(): destination.write(chunk) - # TODO-BLOBSTORE + f.seek(0) + store_file("statement", statement.uploaded_filename, f) else: destination.write(markdown_content) - # TODO-BLOBSTORE + store_str("statement", statement.uploaded_filename, markdown_content) return redirect("ietf.doc.views_doc.document_main", name=statement.name) - else: if statement.uploaded_filename.endswith("pdf"): text = CONST_PDF_REV_NOTICE @@ -256,12 +258,14 @@ def new_statement(request): mode="wb" if writing_pdf else "w" ) as destination: if writing_pdf: - for chunk in form.cleaned_data["statement_file"].chunks(): + f = form.cleaned_data["statement_file"] + for chunk in f.chunks(): destination.write(chunk) - # TODO-BLOBSTORE + f.seek(0) + store_file("statement", statement.uploaded_filename, f) else: destination.write(markdown_content) - # TODO-BLOBSTORE + store_str("statement", statement.uploaded_filename, markdown_content) return redirect("ietf.doc.views_doc.document_main", name=statement.name) else: diff --git a/ietf/settings.py b/ietf/settings.py index 04d0afec0b..77bd524c95 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -754,6 +754,7 @@ def skip_unreadable_post(record): "bluesheets", "procmaterials", "narrativeminutes", + "statement", ] # Override this in settings_local.py if needed From ff200132b1fd9ded894ae5cc4b7246b00a1337e3 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 24 Jan 2025 16:58:33 -0600 Subject: [PATCH 22/87] feat: store status changes --- ietf/doc/tests_status_change.py | 18 +++++++++++++++--- ietf/doc/views_status_change.py | 8 +++++--- ietf/settings.py | 1 + 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py index bd4da4c092..cbdc1a049a 100644 --- a/ietf/doc/tests_status_change.py +++ b/ietf/doc/tests_status_change.py @@ -19,6 +19,7 @@ WgRfcFactory, DocEventFactory, WgDraftFactory ) from ietf.doc.models import ( Document, State, DocEvent, BallotPositionDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent ) +from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils import create_ballot_if_not_open from ietf.doc.views_status_change import default_approval_text from ietf.group.models import Person @@ -71,7 +72,7 @@ def test_start_review(self): statchg_relation_row_blah="tois") ) self.assertEqual(r.status_code, 302) - status_change = Document.objects.get(name='status-change-imaginary-new') + status_change = Document.objects.get(name='status-change-imaginary-new') self.assertEqual(status_change.get_state('statchg').slug,'adrev') self.assertEqual(status_change.rev,'00') self.assertEqual(status_change.ad.name,'Areað Irector') @@ -563,6 +564,8 @@ def test_initial_submission(self): ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename self.assertFalse(filepath.exists()) self.assertFalse(ftp_filepath.exists()) + with self.assertRaises(FileNotFoundError): + retrieve_str("statchg",basename) r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1")) self.assertEqual(r.status_code,302) doc = Document.objects.get(name='status-change-imaginary-mid-review') @@ -571,6 +574,10 @@ def test_initial_submission(self): self.assertEqual(f.read(),"Some initial review text\n") with ftp_filepath.open() as f: self.assertEqual(f.read(),"Some initial review text\n") + self.assertEqual( + retrieve_str("statchg", basename), + "Some initial review text\n" + ) self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc) def test_subsequent_submission(self): @@ -607,7 +614,8 @@ def test_subsequent_submission(self): self.assertContains(r, "does not appear to be a text file") # sane post uploading a file - test_file = StringIO("This is a new proposal.") + test_content = "This is a new proposal." + test_file = StringIO(test_content) test_file.name = "unnamed" r = self.client.post(url,dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 302) @@ -615,8 +623,12 @@ def test_subsequent_submission(self): self.assertEqual(doc.rev,'01') path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev)) with io.open(path) as f: - self.assertEqual(f.read(),"This is a new proposal.") + self.assertEqual(f.read(), test_content) f.close() + self.assertEqual( + retrieve_str("statchg", f"{doc.name}-{doc.rev}.txt"), + test_content + ) self.assertTrue( "mid-review-01" in doc.latest_event(NewRevisionDocEvent).desc) # verify reset text button works diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index 0f64e2af44..388a97d608 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -26,6 +26,7 @@ BallotPositionDocEvent, NewRevisionDocEvent, WriteupDocEvent, STATUSCHANGE_RELATIONS ) from ietf.doc.forms import AdForm from ietf.doc.lastcall import request_last_call +from ietf.doc.storage_utils import store_str from ietf.doc.utils import add_state_change_event, update_telechat, close_open_ballots, create_ballot_if_not_open from ietf.doc.views_ballot import LastCallTextForm from ietf.group.models import Group @@ -160,10 +161,11 @@ def save(self, doc): filename = Path(settings.STATUS_CHANGE_PATH) / basename with io.open(filename, 'w', encoding='utf-8') as destination: if self.cleaned_data['txt']: - destination.write(self.cleaned_data['txt']) + content = self.cleaned_data['txt'] else: - destination.write(self.cleaned_data['content']) - # TODO-BLOBSTORE + content = self.cleaned_data['content'] + destination.write(content) + store_str("statchg", basename, content) try: ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10 diff --git a/ietf/settings.py b/ietf/settings.py index 77bd524c95..568db2a575 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -755,6 +755,7 @@ def skip_unreadable_post(record): "procmaterials", "narrativeminutes", "statement", + "statchg", ] # Override this in settings_local.py if needed From 863d8ab31d1175b21709371321cb8d5948c70514 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 24 Jan 2025 17:19:17 -0600 Subject: [PATCH 23/87] feat: store liaison attachments --- ietf/liaisons/forms.py | 4 +++- ietf/liaisons/tests.py | 38 +++++++++++++++++++++++++++++--------- ietf/settings.py | 1 + 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 8e6afa3a4f..e2da5abd72 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -21,6 +21,7 @@ import debug # pyflakes:ignore +from ietf.doc.storage_utils import store_file from ietf.ietfauth.utils import has_role from ietf.name.models import DocRelationshipName from ietf.liaisons.utils import get_person_for_user,is_authorized_individual @@ -379,7 +380,8 @@ def save_attachments(self): attach_file = io.open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb') attach_file.write(attached_file.read()) attach_file.close() - # TODO-BLOBSTORE + attached_file.seek(0) + store_file(attach.type_id, attach.uploaded_filename, attached_file) if not self.is_new: # create modified event diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index a0186f6a01..1742687f14 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -19,6 +19,7 @@ from io import StringIO from pyquery import PyQuery +from ietf.doc.storage_utils import retrieve_str from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.mail import outbox @@ -414,7 +415,8 @@ def test_edit_liaison(self): # edit attachments_before = liaison.attachments.count() - test_file = StringIO("hello world") + test_content = "hello world" + test_file = StringIO(test_content) test_file.name = "unnamed" r = self.client.post(url, dict(from_groups=str(from_group.pk), @@ -452,9 +454,12 @@ def test_edit_liaison(self): self.assertEqual(attachment.title, "attachment") with (Path(settings.LIAISON_ATTACH_PATH) / attachment.uploaded_filename).open() as f: written_content = f.read() + self.assertEqual(written_content, test_content) + self.assertEqual( + retrieve_str(attachment.type_id, attachment.uploaded_filename), + test_content, + ) - test_file.seek(0) - self.assertEqual(written_content, test_file.read()) def test_incoming_access(self): '''Ensure only Secretariat, Liaison Managers, and Authorized Individuals @@ -704,7 +709,8 @@ def test_add_incoming_liaison(self): # add new mailbox_before = len(outbox) - test_file = StringIO("hello world") + test_content = "hello world" + test_file = StringIO(test_content) test_file.name = "unnamed" from_groups = [ str(g.pk) for g in Group.objects.filter(type="sdo") ] to_group = Group.objects.get(acronym="mars") @@ -756,6 +762,11 @@ def test_add_incoming_liaison(self): self.assertEqual(attachment.title, "attachment") with (Path(settings.LIAISON_ATTACH_PATH) / attachment.uploaded_filename).open() as f: written_content = f.read() + self.assertEqual(written_content, test_content) + self.assertEqual( + retrieve_str(attachment.type_id, attachment.uploaded_filename), + test_content + ) test_file.seek(0) self.assertEqual(written_content, test_file.read()) @@ -783,7 +794,8 @@ def test_add_outgoing_liaison(self): # add new mailbox_before = len(outbox) - test_file = StringIO("hello world") + test_content = "hello world" + test_file = StringIO(test_content) test_file.name = "unnamed" from_group = Group.objects.get(acronym="mars") to_group = Group.objects.filter(type="sdo")[0] @@ -835,9 +847,11 @@ def test_add_outgoing_liaison(self): self.assertEqual(attachment.title, "attachment") with (Path(settings.LIAISON_ATTACH_PATH) / attachment.uploaded_filename).open() as f: written_content = f.read() - - test_file.seek(0) - self.assertEqual(written_content, test_file.read()) + self.assertEqual(written_content, test_content) + self.assertEqual( + retrieve_str(attachment.type_id, attachment.uploaded_filename), + test_content + ) self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) @@ -882,7 +896,8 @@ def test_liaison_add_attachment(self): # get minimum edit post data - file = StringIO('dummy file') + test_data = "dummy file" + file = StringIO(test_data) file.name = "upload.txt" post_data = dict( from_groups = ','.join([ str(x.pk) for x in liaison.from_groups.all() ]), @@ -909,6 +924,11 @@ def test_liaison_add_attachment(self): self.assertEqual(liaison.attachments.count(),1) event = liaison.liaisonstatementevent_set.order_by('id').last() self.assertTrue(event.desc.startswith('Added attachment')) + attachment = liaison.attachments.get() + self.assertEqual( + retrieve_str(attachment.type_id, attachment.uploaded_filename), + test_data + ) def test_liaison_edit_attachment(self): diff --git a/ietf/settings.py b/ietf/settings.py index 568db2a575..d939d53321 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -756,6 +756,7 @@ def skip_unreadable_post(record): "narrativeminutes", "statement", "statchg", + "liai-att", ] # Override this in settings_local.py if needed From ec114a969eb1e99c840850d66f77fc4ce4f6fe4b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 24 Jan 2025 17:39:57 -0600 Subject: [PATCH 24/87] feat: store agendas provided with Interim session requests --- ietf/meeting/forms.py | 3 ++- ietf/meeting/tests_views.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index 743e33d438..e4cd370e2e 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -20,6 +20,7 @@ import debug # pyflakes:ignore from ietf.doc.models import Document, State, NewRevisionDocEvent +from ietf.doc.storage_utils import store_str from ietf.group.models import Group from ietf.group.utils import groups_managed_by from ietf.meeting.models import Session, Meeting, Schedule, countries, timezones, TimeSlot, Room @@ -361,7 +362,7 @@ def save_agenda(self): os.makedirs(directory) with io.open(path, "w", encoding='utf-8') as file: file.write(self.cleaned_data['agenda']) - # TODO-BLOBSTORE + store_str("agenda", doc.uploaded_filename, self.cleaned_data['agenda']) class InterimAnnounceForm(forms.ModelForm): diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 5ba18cf714..8914ef7126 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -5235,6 +5235,12 @@ def do_interim_request_single_virtual(self, emails_expected): doc = session.materials.first() path = os.path.join(doc.get_file_path(),doc.filename_with_rev()) self.assertTrue(os.path.exists(path)) + with Path(path).open() as f: + self.assertEqual(f.read(), agenda) + self.assertEqual( + retrieve_str("agenda",doc.uploaded_filename), + agenda + ) # check notices to secretariat and chairs self.assertEqual(len(outbox), length_before + emails_expected) return meeting @@ -5302,6 +5308,10 @@ def test_interim_request_single_in_person(self): timeslot = session.official_timeslotassignment().timeslot self.assertEqual(timeslot.time,dt) self.assertEqual(timeslot.duration,duration) + self.assertEqual( + retrieve_str("agenda",session.agenda().uploaded_filename), + agenda + ) def test_interim_request_multi_day(self): make_meeting_test_data() @@ -5369,6 +5379,11 @@ def test_interim_request_multi_day(self): self.assertEqual(timeslot.time,dt2) self.assertEqual(timeslot.duration,duration) self.assertEqual(session.agenda_note,agenda_note) + for session in meeting.session_set.all(): + self.assertEqual( + retrieve_str("agenda",session.agenda().uploaded_filename), + agenda + ) def test_interim_request_multi_day_non_consecutive(self): make_meeting_test_data() @@ -5518,6 +5533,11 @@ def test_interim_request_series(self): self.assertEqual(timeslot.time,dt2) self.assertEqual(timeslot.duration,duration) self.assertEqual(session.agenda_note,agenda_note) + for session in meeting.session_set.all(): + self.assertEqual( + retrieve_str("agenda",session.agenda().uploaded_filename), + agenda + ) # test_interim_pending subsumed by test_appears_on_pending @@ -6091,6 +6111,10 @@ def test_interim_request_edit_agenda_updates_doc(self): self.assertNotEqual(agenda_doc.uploaded_filename, uploaded_filename_before, 'Uploaded filename should be updated') with (Path(agenda_doc.get_file_path()) / agenda_doc.uploaded_filename).open() as f: self.assertEqual(f.read(), 'modified agenda contents', 'New agenda contents should be saved') + self.assertEqual( + retrieve_str(agenda_doc.type_id, agenda_doc.uploaded_filename), + "modified agenda contents" + ) def test_interim_request_details_permissions(self): make_interim_test_data() From 1848cd1cad593878b1fdba276f6eae52563a6d86 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Sat, 25 Jan 2025 14:24:52 -0600 Subject: [PATCH 25/87] chore: capture TODOs --- ietf/meeting/models.py | 2 +- ietf/meeting/utils.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index ecc620cb65..a0c096464a 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1431,7 +1431,7 @@ class MeetingHost(models.Model): """Meeting sponsor""" meeting = ForeignKey(Meeting, related_name='meetinghosts') name = models.CharField(max_length=255, blank=False) - # TODO-BLOBSTORE - capture these logos + # TODO-BLOBSTORE - capture these logos and look for other ImageField like model fields. logo = MissingOkImageField( storage=NoLocationMigrationFileSystemStorage(location=settings.MEETINGHOST_LOGO_PATH), upload_to=_host_upload_path, diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 5b3badbee9..f1da004a3d 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -226,11 +226,12 @@ def generate_bluesheet(request, session): 'session': session, 'data': data, }) + # TODO-BLOBSTORE Verify that this is only creating a file-like object to pass along + # if so, we can do this in memory and not involve disk. fd, name = tempfile.mkstemp(suffix=".txt", text=True) os.close(fd) with open(name, "w") as file: file.write(text) - # TODO-BLOBSTORE with open(name, "br") as file: return save_bluesheet(request, session, file) From ae636022f7cb6b7db4d3e6ab50a19d3e439cd79b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Sat, 25 Jan 2025 14:43:48 -0600 Subject: [PATCH 26/87] feat: store polls and chatlogs --- ietf/api/tests.py | 5 +++++ ietf/doc/storage_utils.py | 7 +++---- ietf/meeting/utils.py | 6 +++--- ietf/settings.py | 2 ++ 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index d9af457e95..ac0b37a608 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -25,6 +25,7 @@ import debug # pyflakes:ignore import ietf +from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils import get_unicode_document_content from ietf.doc.models import RelatedDocument, State from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory @@ -553,6 +554,10 @@ def test_api_upload_polls_and_chatlog(self): newdoc = session.presentations.get(document__type_id=type_id).document newdoccontent = get_unicode_document_content(newdoc.name, Path(session.meeting.get_materials_path()) / type_id / newdoc.uploaded_filename) self.assertEqual(json.loads(content), json.loads(newdoccontent)) + self.assertEqual( + json.loads(retrieve_str(type_id, newdoc.uploaded_filename)), + json.loads(content) + ) def test_api_upload_bluesheet(self): url = urlreverse("ietf.meeting.views.api_upload_bluesheet") diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 022dfa1136..bce48e7fb1 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -32,10 +32,9 @@ def store_file(kind: str, name: str, file: File, allow_overwrite: bool = False) debug.show("e") return None if new_name != name: - log( - f"Conflict encountered saving {name} - results stored in {new_name} instead." - ) - debug.show('f"Conflict encountered saving {name} - results stored in {new_name} instead."') + complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." + log(complaint) + debug.show("complaint") return None diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index f1da004a3d..ec0497fa73 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -21,7 +21,7 @@ import debug # pyflakes:ignore from ietf.dbtemplate.models import DBTemplate -from ietf.doc.storage_utils import store_bytes +from ietf.doc.storage_utils import store_bytes, store_str from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, Constraint, SchedTimeSessAssignment, SessionPresentation, Attended) from ietf.doc.models import Document, State, NewRevisionDocEvent @@ -827,8 +827,8 @@ def write_doc_for_session(session, type_id, filename, contents): path.mkdir(parents=True, exist_ok=True) with open(path / filename, "wb") as file: file.write(contents.encode('utf-8')) - # TODO-BLOBSTORE - return + store_str(type_id, filename.name, contents) + return None def create_recording(session, url, title=None, user=None): ''' diff --git a/ietf/settings.py b/ietf/settings.py index d939d53321..977593187f 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -757,6 +757,8 @@ def skip_unreadable_post(record): "statement", "statchg", "liai-att", + "chatlog", + "polls", ] # Override this in settings_local.py if needed From ce663e5b239ab527da365a3bdb133a73df78fd15 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Sat, 25 Jan 2025 14:46:55 -0600 Subject: [PATCH 27/87] chore: remove unneeded TODO --- ietf/meeting/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index e5508e604c..45a162e6e9 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -4699,8 +4699,6 @@ def err(code, text): save_err = save_bluesheet(request, session, file) if save_err: return err(400, save_err) - # TODO-BLOBSTORE - return HttpResponse("Done", status=200, content_type='text/plain') From be91813f5d7a7b518f91a58a3c7a5b8954243a29 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 27 Jan 2025 16:57:53 -0600 Subject: [PATCH 28/87] feat: store drafts on submit and post --- ietf/doc/expire.py | 7 +++++ ietf/doc/storage_utils.py | 25 +++++++++++++++ ietf/settings.py | 3 ++ ietf/submit/tests.py | 65 ++++++++++++++++++++++++++++++++++++--- ietf/submit/utils.py | 23 +++++++++----- 5 files changed, 111 insertions(+), 12 deletions(-) diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index 98554bae0e..40366bb7dd 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -13,6 +13,7 @@ from typing import List, Optional # pyflakes:ignore +from ietf.doc.storage_utils import remove_from_storage from ietf.doc.utils import update_action_holders from ietf.utils import log from ietf.utils.mail import send_mail @@ -156,11 +157,17 @@ def remove_ftp_copy(f): if mark.exists(): mark.unlink() + def remove_from_active_draft_storage(file): + # Assumes the glob will never find a file with no suffix + ext = file.suffix[1:] + remove_from_storage("active-draft", f"{ext}/{file.name}") + # Note that the object is already in the "draft" storage. src_dir = Path(settings.INTERNET_DRAFT_PATH) for file in src_dir.glob("%s-%s.*" % (doc.name, rev)): move_file(str(file.name)) remove_ftp_copy(str(file.name)) + remove_from_active_draft_storage(file) def expire_draft(doc): # clean up files diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index bce48e7fb1..b6b59d6de9 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -17,7 +17,32 @@ def _get_storage(kind: str) -> Storage: raise NotImplementedError(f"Don't know how to store {kind}") +def exists_in_storage(kind: str, name: str) -> bool: + store = _get_storage(kind) + try: + # open is realized with a HEAD + # See https://github.com/jschneier/django-storages/blob/b79ea310201e7afd659fe47e2882fe59aae5b517/storages/backends/s3.py#L528 + with store.open(name): + return True + except FileNotFoundError: + return False + + +def remove_from_storage(kind: str, name: str) -> None: + store = _get_storage(kind) + try: + with store.open(name): + pass + store.delete(name) + # debug.show('f"deleted {name} from {kind} storage"') + except FileNotFoundError: + complaint = f"WARNING: Asked to delete non-existant {name} from {kind} storage" + log(complaint) + # debug.show("complaint") + + def store_file(kind: str, name: str, file: File, allow_overwrite: bool = False) -> None: + # debug.show('f"asked to store {name} into {kind}"') store = _get_storage(kind) if not allow_overwrite and store.exists(name): log(f"Failed to save {kind}:{name} - name already exists in store") diff --git a/ietf/settings.py b/ietf/settings.py index 977593187f..a015850909 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -747,6 +747,7 @@ def skip_unreadable_post(record): "bofreq", "charter", "conflrev", + "active-draft", "draft", "slides", "minutes", @@ -759,6 +760,8 @@ def skip_unreadable_post(record): "liai-att", "chatlog", "polls", + "staging", + "bibxml-ids" ] # Override this in settings_local.py if needed diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 9a79a457db..949b89533a 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -31,7 +31,7 @@ ReviewFactory, WgRfcFactory) from ietf.doc.models import ( Document, DocEvent, State, BallotPositionDocEvent, DocumentAuthor, SubmissionDocEvent ) -from ietf.doc.storage_utils import retrieve_str +from ietf.doc.storage_utils import exists_in_storage, retrieve_str, store_file, store_str from ietf.doc.utils import create_ballot_if_not_open, can_edit_docextresources, update_action_holders from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group @@ -429,7 +429,13 @@ def submit_new_wg(self, formats): self.assertTrue(draft.latest_event(type="added_suggested_replaces")) self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))) self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev)))) - self.assertTrue(len(retrieve_str("draft",f"txt/{name}-{rev}.txt"))>0) + check_ext = ["xml", "txt", "html"] if "xml" in formats else ["txt"] + for ext in check_ext: + basename=f"{name}-{rev}.{ext}" + extname=f"{ext}/{basename}" + self.assertFalse(exists_in_storage("staging", basename)) + self.assertTrue(exists_in_storage("active-draft", extname)) + self.assertTrue(exists_in_storage("draft", extname)) self.assertEqual(draft.type_id, "draft") self.assertEqual(draft.stream_id, "ietf") self.assertTrue(draft.expires >= timezone.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1)) @@ -773,6 +779,13 @@ def inspect_docevents(docevents, event_delta, event_type, be_in_desc, by_name): self.assertTrue(os.path.exists(os.path.join(self.archive_dir, "%s-%s.txt" % (name, old_rev)))) self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))) self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev)))) + check_ext = ["xml", "txt", "html"] if "xml" in formats else ["txt"] + for ext in check_ext: + basename=f"{name}-{rev}.{ext}" + extname=f"{ext}/{basename}" + self.assertFalse(exists_in_storage("staging", basename)) + self.assertTrue(exists_in_storage("active-draft", extname)) + self.assertTrue(exists_in_storage("draft", extname)) self.assertEqual(draft.type_id, "draft") if stream_type == 'ietf': self.assertEqual(draft.stream_id, "ietf") @@ -973,7 +986,13 @@ def submit_new_individual(self, formats): self.assertTrue(variant_path.samefile(variant_ftp_path)) variant_all_archive_path = Path(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR) / variant_path.name self.assertTrue(variant_path.samefile(variant_all_archive_path)) - + check_ext = ["xml", "txt", "html"] if "xml" in formats else ["txt"] + for ext in check_ext: + basename=f"{name}-{rev}.{ext}" + extname=f"{ext}/{basename}" + self.assertFalse(exists_in_storage("staging", basename)) + self.assertTrue(exists_in_storage("active-draft", extname)) + self.assertTrue(exists_in_storage("draft", extname)) def test_submit_new_individual_txt(self): @@ -1418,6 +1437,7 @@ def test_cancel_submission(self): # cancel r = self.client.post(status_url, dict(action=action)) self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))) + self.assertFalse(exists_in_storage("staging",f"{name}-{rev}.txt")) def test_edit_submission_and_force_post(self): # submit -> edit @@ -1607,16 +1627,21 @@ def test_submit_all_file_types(self): self.assertEqual(Submission.objects.filter(name=name).count(), 1) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))) + self.assertTrue(exists_in_storage("staging",f"{name}-{rev}.txt")) fd = io.open(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))) txt_contents = fd.read() fd.close() self.assertTrue(name in txt_contents) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev)))) + self.assertTrue(exists_in_storage("staging",f"{name}-{rev}.txt")) fd = io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))) xml_contents = fd.read() fd.close() self.assertTrue(name in xml_contents) self.assertTrue('' in xml_contents) + xml_contents = retrieve_str("staging", f"{name}-{rev}.xml") + self.assertTrue(name in xml_contents) + self.assertTrue('' in xml_contents) def test_expire_submissions(self): s = Submission.objects.create(name="draft-ietf-mars-foo", @@ -2767,10 +2792,13 @@ def test_process_and_accept_uploaded_submission(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(xml_data) + store_str("submission", "draft-somebpdy-test-00.xml", xml_data) txt_path = xml_path.with_suffix('.txt') self.assertFalse(txt_path.exists()) html_path = xml_path.with_suffix('.html') self.assertFalse(html_path.exists()) + for ext in ["txt", "html"]: + self.assertFalse(exists_in_storage("staging",f"draft-somebody-test-00.{ext}")) process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh @@ -2786,6 +2814,8 @@ def test_process_and_accept_uploaded_submission(self): # at least test that these were created self.assertTrue(txt_path.exists()) self.assertTrue(html_path.exists()) + for ext in ["txt", "html"]: + self.assertFalse(exists_in_storage("staging",f"draft-somebody-test-00.{ext}")) self.assertEqual(submission.file_size, os.stat(txt_path).st_size) self.assertIn('Completed submission validation checks', submission.submissionevent_set.last().desc) @@ -2811,6 +2841,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(xml_data) + store_str("staging", "draft-somebody-test-00.xml", xml_data) process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh self.assertEqual(submission.state_id, 'cancel') @@ -2827,6 +2858,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(re.sub(r'.*', '', xml_data)) + store_str("staging", "draft-somebody-test-00.xml", re.sub(r'.*', '', xml_data)) process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh self.assertEqual(submission.state_id, 'cancel') @@ -2843,6 +2875,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(re.sub(r'.*', '', xml_data)) + store_str("staging", "draft-somebody-test-00.xml", re.sub(r'.*', '', xml_data)) process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh self.assertEqual(submission.state_id, 'cancel') @@ -2859,6 +2892,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-different-name-00.xml' with xml_path.open('w') as f: f.write(xml_data) + store_str("staging", "draft-different-name-00.xml", xml_data) process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh self.assertEqual(submission.state_id, 'cancel') @@ -2875,6 +2909,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-01.xml' with xml_path.open('w') as f: f.write(xml_data) + store_str("staging", "draft-somebody-test-01.xml", xml_data) process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh self.assertEqual(submission.state_id, 'cancel') @@ -2891,6 +2926,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.txt' with txt_path.open('w') as f: f.write(txt_data) + store_str("staging", "draft-somebody-test-00.txt", txt_data) process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh self.assertEqual(submission.state_id, 'cancel') @@ -2905,8 +2941,9 @@ def test_process_and_accept_uploaded_submission_invalid(self): state_id='uploaded', ) xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' - with xml_path.open('w') as f: + with xml_path.open('w') as f: # Why is this state being written if the thing that uses it is mocked out? f.write(xml_data) + store_str("staging", "draft-somebody-test-00.xml", xml_data) with mock.patch('ietf.submit.utils.process_submission_xml') as mock_proc_xml: process_and_accept_uploaded_submission(submission) submission = Submission.objects.get(pk=submission.pk) # refresh @@ -2924,6 +2961,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(xml_data) + store_str("staging", "draft-somebody-test-00.xml", xml_data) with mock.patch( 'ietf.submit.utils.apply_checkers', side_effect = lambda _, __: submission.checks.create( @@ -2970,6 +3008,7 @@ def test_process_submission_xml(self): ) xml_contents = xml.read() xml_path.write_text(xml_contents) + store_str("staging", "draft-somebody-test-00.xml", xml_contents) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["filename"], "draft-somebody-test") self.assertEqual(output["rev"], "00") @@ -2986,18 +3025,22 @@ def test_process_submission_xml(self): # Should behave on missing or partial elements xml_path.write_text(re.sub(r"", "", xml_contents)) # strip entirely + store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", "", xml_contents)) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], None) xml_path.write_text(re.sub(r")", r"\1 day=\2", xml_contents)) # remove month + store_str("staging", "draft-somebody-test-00.xml", re.sub(r"()", r"\1 day=\2", xml_contents)) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], date_today()) xml_path.write_text(re.sub(r"", r"", xml_contents)) # remove day + store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", r"", xml_contents)) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], date_today()) @@ -3010,6 +3053,8 @@ def test_process_submission_xml(self): title="Correct Draft Title", ) xml_path.write_text(xml.read()) + xml.seek(0) + store_file("staging", "draft-somebody-test-00.xml", xml) with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"): process_submission_xml("draft-somebody-test", "00") @@ -3022,6 +3067,8 @@ def test_process_submission_xml(self): title="Correct Draft Title", ) xml_path.write_text(xml.read()) + xml.seek(0) + store_file("staging", "draft-somebody-test-00.xml", xml) with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"): process_submission_xml("draft-somebody-test", "00") @@ -3034,6 +3081,8 @@ def test_process_submission_xml(self): title="", ) xml_path.write_text(xml.read()) + xml.seek(0) + store_file("staging", "draft-somebody-test-00.xml", xml) with self.assertRaisesMessage(SubmissionError, "Could not extract a valid title"): process_submission_xml("draft-somebody-test", "00") @@ -3047,6 +3096,8 @@ def test_process_submission_text(self): title="Correct Draft Title", ) txt_path.write_text(txt.read()) + txt.seek(0) + store_file("staging", "draft-somebody-test-00.txt", txt) output = process_submission_text("draft-somebody-test", "00") self.assertEqual(output["filename"], "draft-somebody-test") self.assertEqual(output["rev"], "00") @@ -3071,6 +3122,8 @@ def test_process_submission_text(self): ) with txt_path.open('w') as fd: fd.write(txt.read()) + txt.seek(0) + store_file("staging", "draft-somebody-test-00.txt", txt) txt.close() with self.assertRaisesMessage(SubmissionError, 'disagrees with submission filename'): process_submission_text("draft-somebody-test", "00") @@ -3085,6 +3138,8 @@ def test_process_submission_text(self): ) with txt_path.open('w') as fd: fd.write(txt.read()) + txt.seek(0) + store_file("staging", "draft-somebody-test-00.txt", txt) txt.close() with self.assertRaisesMessage(SubmissionError, 'disagrees with submission revision'): process_submission_text("draft-somebody-test", "00") @@ -3223,6 +3278,7 @@ def test_find_submission_filenames(self): path = Path(self.staging_dir) for ext in ['txt', 'xml', 'pdf', 'md']: (path / f'{draft.name}-{draft.rev}.{ext}').touch() + store_str("staging", f"{draft.name}-{draft.rev}.{ext}", "") files = find_submission_filenames(draft) self.assertCountEqual( files, @@ -3282,6 +3338,7 @@ def test_validate_submission_rev(self): new_wg_doc = WgDraftFactory(rev='01', relations=[('replaces',old_wg_doc)]) path = Path(self.archive_dir) / f'{new_wg_doc.name}-{new_wg_doc.rev}.txt' path.touch() + store_str("staging", f"{new_wg_doc.name}-{new_wg_doc.rev}.txt", "") bad_revs = (None, '', '2', 'aa', '00', '01', '100', '002', u'öö') for rev in bad_revs: diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 2a050bb7dd..69bef0db75 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -36,7 +36,7 @@ DocumentAuthor, AddedMessageEvent ) from ietf.doc.models import NewRevisionDocEvent from ietf.doc.models import RelatedDocument, DocRelationshipName, DocExtResource -from ietf.doc.storage_utils import store_bytes +from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_bytes, store_file, store_str from ietf.doc.utils import (add_state_change_event, rebuild_reference_relations, set_replaces_for_document, prettify_std_name, update_doc_extresources, can_edit_docextresources, update_documentauthors, update_action_holders, @@ -499,7 +499,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev )) with io.open(ref_rev_file_name, "w", encoding='utf-8') as f: f.write(ref_text) - # TODO-BLOBSTORE + store_str("bibxml-ids", f"reference.I-D.{draft.name}-{draft.rev}.txt", ref_text) # TODO-BLOBSTORE verify with test log.log(f"{submission.name}: done") @@ -667,9 +667,12 @@ def move_files_to_repository(submission): ftp_dest = Path(settings.FTP_DIR) / "internet-drafts" / dest.name os.link(dest, all_archive_dest) os.link(dest, ftp_dest) - with open(dest,"rb") as f: - content_bytes = f.read() - store_bytes("draft", f"{ext}/{fname}", content_bytes) + # Shadow what's happening to the fs in the blobstores. When the stores become + # authoritative, the source and dest checks will need to apply to the stores instead. + content_bytes = retrieve_bytes("staging", fname) + store_bytes("active-draft", f"{ext}/{fname}", content_bytes) + store_bytes("draft", f"{ext}/{fname}", content_bytes) + remove_from_storage("staging", fname) elif dest.exists(): log.log("Intended to move '%s' to '%s', but found source missing while destination exists.") elif f".{ext}" in submission.file_types.split(','): @@ -686,6 +689,7 @@ def remove_staging_files(name, rev, exts=None): basename = pathlib.Path(settings.IDSUBMIT_STAGING_PATH) / f'{name}-{rev}' for ext in exts: basename.with_suffix(ext).unlink(missing_ok=True) + remove_from_storage("staging", basename.with_suffix(ext).name) def remove_submission_files(submission): @@ -774,7 +778,8 @@ def save_files(form): for chunk in f.chunks(): destination.write(chunk) log.log("saved file %s" % name) - # TODO-BLOBSTORE + f.seek(0) + store_file("staging", f"{form.filename}-{form.revision}.{ext}", f) return file_name @@ -997,7 +1002,8 @@ def render_missing_formats(submission): xml_version, ) ) - # TODO-BLOBSTORE + with Path(txt_path).open() as f: + store_file("staging", f"{submission.name}-{submission.rev}.txt", f) # --- Convert to html --- html_path = staging_path(submission.name, submission.rev, '.html') @@ -1020,7 +1026,8 @@ def render_missing_formats(submission): xml_version, ) ) - # TODO-BLOBSTORE + with Path(html_path).open() as f: + store_file("staging", f"{submission.name}-{submission.rev}.html", f) def accept_submission(submission: Submission, request: Optional[HttpRequest] = None, autopost=False): From 639020541831f1389d518d14da56dee3efad27d1 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 27 Jan 2025 17:44:17 -0600 Subject: [PATCH 29/87] fix: handle storage during doc expiration and resurrection --- ietf/doc/expire.py | 9 ++++++++- ietf/doc/tests_draft.py | 13 +++++++++++++ ietf/doc/views_draft.py | 6 ++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index 40366bb7dd..c52e0679aa 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -13,7 +13,7 @@ from typing import List, Optional # pyflakes:ignore -from ietf.doc.storage_utils import remove_from_storage +from ietf.doc.storage_utils import exists_in_storage, remove_from_storage from ietf.doc.utils import update_action_holders from ietf.utils import log from ietf.utils.mail import send_mail @@ -225,6 +225,13 @@ def move_file_to(subdir): mark = Path(settings.FTP_DIR) / "internet-drafts" / basename if mark.exists(): mark.unlink() + if ext: + # Note that we're not moving these strays anywhere - the assumption + # is that the active-draft blobstore will not get strays. + # See, however, the note about "major system failures" at "unknown_ids" + blobname = f"{ext[1:]}/{basename}" + if exists_in_storage("active-draft", blobname): + remove_from_storage("active-draft", blobname) try: doc = Document.objects.get(name=filename, rev=revision) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 2405806682..d0a45f3723 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -24,6 +24,7 @@ from ietf.doc.models import ( Document, DocReminder, DocEvent, ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent ) +from ietf.doc.storage_utils import exists_in_storage, store_str from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open from ietf.doc.views_draft import AdoptDraftForm from ietf.name.models import DocTagName, RoleName @@ -577,6 +578,11 @@ def setUp(self): def write_draft_file(self, name, size): with (Path(settings.INTERNET_DRAFT_PATH) / name).open('w') as f: f.write("a" * size) + _, ext = os.path.splitext(name) + if ext: + ext=ext[1:] + store_str("active-draft", f"{ext}/{name}", "a"*size) + store_str("draft", f"{ext}/{name}", "a"*size) class ResurrectTests(DraftFileMixin, TestCase): @@ -649,6 +655,7 @@ def test_resurrect(self): # ensure file restored from archive directory self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt))) self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt))) + self.assertTrue(exists_in_storage("active-draft",f"txt/{txt}")) class ExpireIDsTests(DraftFileMixin, TestCase): @@ -775,6 +782,7 @@ def test_expire_drafts(self): self.assertEqual(draft.action_holders.count(), 0) self.assertIn('Removed all action holders', draft.latest_event(type='changed_action_holders').desc) self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt))) + self.assertFalse(exists_in_storage("active-draft", f"txt/{txt}")) self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt))) draft.delete() @@ -798,6 +806,7 @@ def test_clean_up_draft_files(self): clean_up_draft_files() self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, unknown))) + self.assertFalse(exists_in_storage("active-draft", f"txt/{unknown}")) self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, "unknown_ids", unknown))) @@ -808,6 +817,7 @@ def test_clean_up_draft_files(self): clean_up_draft_files() self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, malformed))) + self.assertFalse(exists_in_storage("active-draft", f"txt/{malformed}")) self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, "unknown_ids", malformed))) @@ -822,9 +832,11 @@ def test_clean_up_draft_files(self): clean_up_draft_files() self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt))) + self.assertFalse(exists_in_storage("active-draft", f"txt/{txt}")) self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt))) self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, pdf))) + self.assertFalse(exists_in_storage("active-draft", f"pdf/{pdf}")) self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, pdf))) # expire draft @@ -843,6 +855,7 @@ def test_clean_up_draft_files(self): clean_up_draft_files() self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt))) + self.assertFalse(exists_in_storage("active-draft", f"txt/{txt}")) self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt))) diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 34104b2005..c80537afb3 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -32,6 +32,7 @@ generate_publication_request, email_adopted, email_intended_status_changed, email_iesg_processing_document, email_ad_approved_doc, email_iana_expert_review_state_changed ) +from ietf.doc.storage_utils import retrieve_bytes, store_bytes from ietf.doc.utils import ( add_state_change_event, can_adopt_draft, can_unadopt_draft, get_tags_for_stream_id, nice_consensus, update_action_holders, update_reminder, update_telechat, make_notify_changed_event, get_initial_notify, @@ -897,6 +898,11 @@ def restore_draft_file(request, draft): except shutil.Error as ex: messages.warning(request, 'There was an error restoring the Internet-Draft file: {} ({})'.format(file, ex)) log.log(" Exception %s when attempting to move %s" % (ex, file)) + _, ext = os.path.splitext(os.path.basename(file)) + if ext: + ext = ext[1:] + blobname = f"{ext}/{basename}.{ext}" + store_bytes("active-draft", blobname, retrieve_bytes("draft", blobname)) class ShepherdWriteupUploadForm(forms.Form): From 2672de37ebee6be5b2874ca9e5f5bcddba5a2c97 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 28 Jan 2025 09:30:40 -0600 Subject: [PATCH 30/87] fix: mirror an unlink --- ietf/meeting/tests_views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 8914ef7126..dd223e403d 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -37,7 +37,7 @@ import debug # pyflakes:ignore from ietf.doc.models import Document, NewRevisionDocEvent -from ietf.doc.storage_utils import retrieve_bytes, retrieve_str +from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, retrieve_str from ietf.group.models import Group, Role, GroupFeatures from ietf.group.utils import can_manage_group from ietf.person.models import Person @@ -7191,7 +7191,9 @@ def test_handles_missing_previous_revision_file(self): # remove the file uploaded for the first rev minutes_docs = self.session.presentations.filter(document__type='minutes') self.assertEqual(minutes_docs.count(), 1) - Path(minutes_docs.first().document.get_file_name()).unlink() + to_remove = Path(minutes_docs.first().document.get_file_name()) + to_remove.unlink() + remove_from_storage("minutes", to_remove.name) self.assertEqual(r.status_code, 302) with requests_mock.Mocker() as mock: From 7065bf45151cfc328de0fb6afa4bdc529962cea9 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 28 Jan 2025 09:40:29 -0600 Subject: [PATCH 31/87] chore: add/refine TODOs --- ietf/idindex/tasks.py | 3 ++- ietf/meeting/views.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ietf/idindex/tasks.py b/ietf/idindex/tasks.py index 2abec2bf5e..cd8e662936 100644 --- a/ietf/idindex/tasks.py +++ b/ietf/idindex/tasks.py @@ -39,7 +39,8 @@ def move_into_place(self, src_path: Path, dest_path: Path, hardlink_dirs: List[P target.unlink(missing_ok=True) os.link(dest_path, target) # until python>=3.10 - # TODO-BLOBSTORE : Going to need something to put these generated things into storage + # TODO-BLOBSTORE : Going to need something to put these generated things into storage? + # Or are we doing this in parallel with each use of TempFileManager in the code? def cleanup(self): for tf_path in self.cleanup_list: diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 45a162e6e9..469f56c3fc 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -5010,6 +5010,7 @@ def approve_proposed_slides(request, slidesubmission_id, num): if not os.path.exists(path): os.makedirs(path) shutil.move(submission.staged_filepath(), os.path.join(path, target_filename)) + # TODO-BLOBSTORE post_process(doc) DocEvent.objects.create(type="approved_slides", doc=doc, rev=doc.rev, by=request.user.person, desc="Slides approved") @@ -5050,6 +5051,7 @@ def approve_proposed_slides(request, slidesubmission_id, num): try: if submission.filename != None and submission.filename != '': os.unlink(submission.staged_filepath()) + # TODO-BLOBSTORE except (FileNotFoundError, IsADirectoryError): pass acronym = submission.session.group.acronym From 2946cdb964a462897b5b1d8f8abd00aaecdbe156 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 28 Jan 2025 11:22:53 -0600 Subject: [PATCH 32/87] feat: store slide submissions --- ietf/meeting/factories.py | 5 +++++ ietf/meeting/tests_views.py | 34 ++++++++++++++++++++++++++++++---- ietf/meeting/views.py | 19 ++++++++++++------- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/ietf/meeting/factories.py b/ietf/meeting/factories.py index 69c1f0421b..eb36e9e756 100644 --- a/ietf/meeting/factories.py +++ b/ietf/meeting/factories.py @@ -9,6 +9,7 @@ from django.core.files.base import ContentFile from django.db.models import Q +from ietf.doc.storage_utils import store_str from ietf.meeting.models import (Attended, Meeting, Session, SchedulingEvent, Schedule, TimeSlot, SessionPresentation, FloorPlan, Room, SlideSubmission, Constraint, MeetingHost, ProceedingsMaterial) @@ -239,6 +240,10 @@ class Meta: make_file = factory.PostGeneration( lambda obj, create, extracted, **kwargs: open(obj.staged_filepath(),'a').close() ) + + store_submission = factory.PostGeneration( + lambda obj, create, extracted, **kwargs: store_str("staging", obj.filename, "") + ) class ConstraintFactory(factory.django.DjangoModelFactory): class Meta: diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index dd223e403d..eeb44a5b17 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -37,7 +37,7 @@ import debug # pyflakes:ignore from ietf.doc.models import Document, NewRevisionDocEvent -from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, retrieve_str +from ietf.doc.storage_utils import exists_in_storage, remove_from_storage, retrieve_bytes, retrieve_str from ietf.group.models import Group, Role, GroupFeatures from ietf.group.utils import can_manage_group from ietf.person.models import Person @@ -6785,7 +6785,6 @@ def test_remove_sessionpresentation(self, mock_slides_manager_cls): self.assertEqual(2, agenda.docevent_set.count()) self.assertFalse(mock_slides_manager_cls.called) - # TODO-BLOBSTORE - review proposed slides def test_propose_session_slides(self): for type_id in ['ietf','interim']: session = SessionFactory(meeting__type_id=type_id) @@ -6812,7 +6811,8 @@ def test_propose_session_slides(self): login_testing_unauthorized(self,newperson.user.username,upload_url) r = self.client.get(upload_url) self.assertEqual(r.status_code,200) - test_file = BytesIO(b'this is not really a slide') + test_bytes = b'this is not really a slide' + test_file = BytesIO(test_bytes) test_file.name = 'not_really.txt' empty_outbox() r = self.client.post(upload_url,dict(file=test_file,title='a test slide file',apply_to_all=True,approved=False)) @@ -6820,6 +6820,11 @@ def test_propose_session_slides(self): session = Session.objects.get(pk=session.pk) self.assertEqual(session.slidesubmission_set.count(),1) self.assertEqual(len(outbox),1) + self.assertFalse(exists_in_storage("slides", session.slidesubmission_set.get().uploaded_filename)) + self.assertEqual( + retrieve_bytes("staging", session.slidesubmission_set.get().uploaded_filename), + test_bytes + ) r = self.client.get(session_overview_url) self.assertEqual(r.status_code, 200) @@ -6839,13 +6844,20 @@ def test_propose_session_slides(self): login_testing_unauthorized(self,chair.user.username,upload_url) r = self.client.get(upload_url) self.assertEqual(r.status_code,200) - test_file = BytesIO(b'this is not really a slide either') + test_bytes = b'this is not really a slide either' + test_file = BytesIO(test_bytes) test_file.name = 'again_not_really.txt' empty_outbox() r = self.client.post(upload_url,dict(file=test_file,title='a selfapproved test slide file',apply_to_all=True,approved=True)) self.assertEqual(r.status_code, 302) self.assertEqual(len(outbox),0) self.assertEqual(session.slidesubmission_set.count(),2) + sp = session.slidesubmission_set.get(title__contains="selfapproved") + self.assertFalse(exists_in_storage("staging", sp.uploaded_filename)) + self.assertEqual( + retrieve_bytes("slides", sp.uploaded_filename), + test_bytes + ) self.client.logout() self.client.login(username=chair.user.username, password=chair.user.username+"+password") @@ -6868,6 +6880,8 @@ def test_disapprove_proposed_slides(self): self.assertEqual(r.status_code,302) self.assertEqual(SlideSubmission.objects.filter(status__slug = 'rejected').count(), 1) self.assertEqual(SlideSubmission.objects.filter(status__slug = 'pending').count(), 0) + if submission.filename is not None and submission.filename != "": + self.assertFalse(exists_in_storage("staging", submission.filename)) r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+rejected") @@ -6886,6 +6900,7 @@ def test_approve_proposed_slides(self, mock_slides_manager_cls): r = self.client.get(url) self.assertEqual(r.status_code,200) empty_outbox() + self.assertTrue(exists_in_storage("staging", submission.filename)) r = self.client.post(url,dict(title='different title',approve='approve')) self.assertEqual(r.status_code,302) self.assertEqual(SlideSubmission.objects.filter(status__slug = 'pending').count(), 0) @@ -6895,6 +6910,8 @@ def test_approve_proposed_slides(self, mock_slides_manager_cls): self.assertIsNotNone(submission.doc) self.assertEqual(session.presentations.count(),1) self.assertEqual(session.presentations.first().document.title,'different title') + self.assertTrue(exists_in_storage("slides", submission.doc.uploaded_filename)) + self.assertFalse(exists_in_storage("staging", submission.filename)) self.assertEqual(mock_slides_manager_cls.call_count, 1) self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings")) self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 1) @@ -6986,12 +7003,15 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): submission = SlideSubmission.objects.get(session=session) + self.assertTrue(exists_in_storage("staging", submission.filename)) approve_url = urlreverse('ietf.meeting.views.approve_proposed_slides', kwargs={'slidesubmission_id':submission.pk,'num':submission.session.meeting.number}) login_testing_unauthorized(self, chair.user.username, approve_url) r = self.client.post(approve_url,dict(title=submission.title,approve='approve')) submission.refresh_from_db() self.assertEqual(r.status_code,302) self.client.logout() + self.assertFalse(exists_in_storage("staging", submission.filename)) + self.assertTrue(exists_in_storage("slides", submission.doc.uploaded_filename)) self.assertEqual(mock_slides_manager_cls.call_count, 1) self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings")) self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 1) @@ -7017,11 +7037,16 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): (first_submission, second_submission) = SlideSubmission.objects.filter(session=session, status__slug = 'pending').order_by('id') + self.assertTrue(exists_in_storage("staging", first_submission.filename)) + self.assertTrue(exists_in_storage("staging", second_submission.filename)) approve_url = urlreverse('ietf.meeting.views.approve_proposed_slides', kwargs={'slidesubmission_id':second_submission.pk,'num':second_submission.session.meeting.number}) login_testing_unauthorized(self, chair.user.username, approve_url) r = self.client.post(approve_url,dict(title=submission.title,approve='approve')) first_submission.refresh_from_db() second_submission.refresh_from_db() + self.assertTrue(exists_in_storage("staging", first_submission.filename)) + self.assertFalse(exists_in_storage("staging", second_submission.filename)) + self.assertTrue(exists_in_storage("slides", second_submission.doc.uploaded_filename)) self.assertEqual(r.status_code,302) self.assertEqual(mock_slides_manager_cls.call_count, 1) self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings")) @@ -7038,6 +7063,7 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): self.assertEqual(r.status_code,302) self.client.logout() self.assertFalse(mock_slides_manager_cls.called) + self.assertFalse(exists_in_storage("staging", first_submission.filename)) self.assertEqual(SlideSubmission.objects.filter(status__slug = 'pending').count(),0) self.assertEqual(SlideSubmission.objects.filter(status__slug = 'rejected').count(),1) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 469f56c3fc..d9b0493126 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -52,6 +52,7 @@ from ietf.doc.fields import SearchableDocumentsField from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent +from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_bytes, store_file from ietf.group.models import Group from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group from ietf.person.models import Person, User @@ -3001,7 +3002,8 @@ def upload_session_slides(request, session_id, num, name=None): for chunk in file.chunks(): destination.write(chunk) destination.close() - # TODO-BLOBSTORE : Do we keep a staging blobstore until we can refactor away exposing staged things through www.ietf.org? + file.seek(0) + store_file("staging", filename, file) submission.filename = filename submission.save() @@ -5010,7 +5012,8 @@ def approve_proposed_slides(request, slidesubmission_id, num): if not os.path.exists(path): os.makedirs(path) shutil.move(submission.staged_filepath(), os.path.join(path, target_filename)) - # TODO-BLOBSTORE + store_bytes("slides", target_filename, retrieve_bytes("staging", submission.filename)) + remove_from_storage("staging", submission.filename) post_process(doc) DocEvent.objects.create(type="approved_slides", doc=doc, rev=doc.rev, by=request.user.person, desc="Slides approved") @@ -5048,12 +5051,14 @@ def approve_proposed_slides(request, slidesubmission_id, num): # in a SlideSubmission object without a file. Handle # this case and keep processing the 'disapprove' even if # the filename doesn't exist. - try: - if submission.filename != None and submission.filename != '': + + if submission.filename != None and submission.filename != '': + try: os.unlink(submission.staged_filepath()) - # TODO-BLOBSTORE - except (FileNotFoundError, IsADirectoryError): - pass + except (FileNotFoundError, IsADirectoryError): + pass + remove_from_storage("staging", submission.filename) + acronym = submission.session.group.acronym submission.status = SlideSubmissionStatusName.objects.get(slug='rejected') submission.save() From c90470508015eeb811e31a78aa6bd334c0d1cc73 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 28 Jan 2025 16:07:37 -0600 Subject: [PATCH 33/87] fix: structure slide test correctly --- ietf/meeting/tests_views.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index eeb44a5b17..66df1d06b9 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -6820,9 +6820,8 @@ def test_propose_session_slides(self): session = Session.objects.get(pk=session.pk) self.assertEqual(session.slidesubmission_set.count(),1) self.assertEqual(len(outbox),1) - self.assertFalse(exists_in_storage("slides", session.slidesubmission_set.get().uploaded_filename)) self.assertEqual( - retrieve_bytes("staging", session.slidesubmission_set.get().uploaded_filename), + retrieve_bytes("staging", session.slidesubmission_set.get().filename), test_bytes ) @@ -6852,10 +6851,10 @@ def test_propose_session_slides(self): self.assertEqual(r.status_code, 302) self.assertEqual(len(outbox),0) self.assertEqual(session.slidesubmission_set.count(),2) - sp = session.slidesubmission_set.get(title__contains="selfapproved") - self.assertFalse(exists_in_storage("staging", sp.uploaded_filename)) + sp = session.presentations.get(document__title__contains="selfapproved") + self.assertFalse(exists_in_storage("staging", sp.document.uploaded_filename)) self.assertEqual( - retrieve_bytes("slides", sp.uploaded_filename), + retrieve_bytes("slides", sp.document.uploaded_filename), test_bytes ) self.client.logout() From 6fac1982a4cfa3ba9c1bdc93bedbd9062fecf1ab Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 28 Jan 2025 16:17:45 -0600 Subject: [PATCH 34/87] fix: correct sense of existence check --- ietf/submit/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 949b89533a..faaa038b32 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -2792,7 +2792,7 @@ def test_process_and_accept_uploaded_submission(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(xml_data) - store_str("submission", "draft-somebpdy-test-00.xml", xml_data) + store_str("staging", "draft-somebpdy-test-00.xml", xml_data) txt_path = xml_path.with_suffix('.txt') self.assertFalse(txt_path.exists()) html_path = xml_path.with_suffix('.html') @@ -2815,7 +2815,7 @@ def test_process_and_accept_uploaded_submission(self): self.assertTrue(txt_path.exists()) self.assertTrue(html_path.exists()) for ext in ["txt", "html"]: - self.assertFalse(exists_in_storage("staging",f"draft-somebody-test-00.{ext}")) + self.assertTrue(exists_in_storage("staging", f"draft-somebody-test-00.{ext}")) self.assertEqual(submission.file_size, os.stat(txt_path).st_size) self.assertIn('Completed submission validation checks', submission.submissionevent_set.last().desc) From 90f04a951f48dd8afdb76346bbdab628a5b57572 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 28 Jan 2025 16:51:40 -0600 Subject: [PATCH 35/87] feat: store some indexes --- ietf/doc/storage_utils.py | 4 +++- ietf/idindex/tasks.py | 7 ++++--- ietf/idindex/tests.py | 5 +++++ ietf/settings.py | 3 ++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index b6b59d6de9..9b446ff57f 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -1,5 +1,7 @@ # Copyright The IETF Trust 2025, All Rights Reserved +from io import BufferedReader +from typing import Union import debug # pyflakes ignore from django.conf import settings @@ -41,7 +43,7 @@ def remove_from_storage(kind: str, name: str) -> None: # debug.show("complaint") -def store_file(kind: str, name: str, file: File, allow_overwrite: bool = False) -> None: +def store_file(kind: str, name: str, file: Union[File,BufferedReader], allow_overwrite: bool = False) -> None: # debug.show('f"asked to store {name} into {kind}"') store = _get_storage(kind) if not allow_overwrite and store.exists(name): diff --git a/ietf/idindex/tasks.py b/ietf/idindex/tasks.py index cd8e662936..e1ea83910b 100644 --- a/ietf/idindex/tasks.py +++ b/ietf/idindex/tasks.py @@ -15,6 +15,8 @@ from django.conf import settings +from ietf.doc.storage_utils import store_file + from .index import all_id_txt, all_id2_txt, id_index_txt @@ -38,9 +40,8 @@ def move_into_place(self, src_path: Path, dest_path: Path, hardlink_dirs: List[P target = path / dest_path.name target.unlink(missing_ok=True) os.link(dest_path, target) # until python>=3.10 - - # TODO-BLOBSTORE : Going to need something to put these generated things into storage? - # Or are we doing this in parallel with each use of TempFileManager in the code? + with dest_path.open("rb") as f: + store_file("indexes", dest_path.name, f) def cleanup(self): for tf_path in self.cleanup_list: diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index 44abf805f0..5cc7a7b3bb 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -15,6 +15,7 @@ from ietf.doc.factories import WgDraftFactory, RfcFactory from ietf.doc.models import Document, RelatedDocument, State, LastCallDocEvent, NewRevisionDocEvent +from ietf.doc.storage_utils import retrieve_str from ietf.group.factories import GroupFactory from ietf.name.models import DocRelationshipName from ietf.idindex.index import all_id_txt, all_id2_txt, id_index_txt @@ -203,5 +204,9 @@ def test_temp_file_manager(self): self.assertFalse(path2.exists()) # left behind # check destination contents and permissions self.assertEqual(dest.read_text(), "yay") + self.assertEqual( + retrieve_str("indexes", "yay.txt"), + "yay" + ) self.assertEqual(dest.stat().st_mode & 0o777, 0o644) self.assertTrue(dest.samefile(other_path / "yay.txt")) diff --git a/ietf/settings.py b/ietf/settings.py index a015850909..4e41364545 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -761,7 +761,8 @@ def skip_unreadable_post(record): "chatlog", "polls", "staging", - "bibxml-ids" + "bibxml-ids", + "indexes" ] # Override this in settings_local.py if needed From 790fa53e0b1bfde0f955f16915518dc851229059 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 16:51:42 -0400 Subject: [PATCH 36/87] feat: BlobShadowFileSystemStorage --- ietf/doc/storage_utils.py | 2 +- ietf/utils/storage.py | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index b6b59d6de9..119d115b3c 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -61,7 +61,7 @@ def store_file(kind: str, name: str, file: File, allow_overwrite: bool = False) log(complaint) debug.show("complaint") return None - + # TODO return value on other paths def store_bytes( kind: str, name: str, content: bytes, allow_overwrite: bool = False diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py index 0aa02cab86..148224c3c6 100644 --- a/ietf/utils/storage.py +++ b/ietf/utils/storage.py @@ -1,8 +1,42 @@ +# Copyright The IETF Trust 2020-2025, All Rights Reserved +"""Django Storage classes""" from django.core.files.storage import FileSystemStorage +from ietf.doc.storage_utils import store_file +from .log import log + class NoLocationMigrationFileSystemStorage(FileSystemStorage): - def deconstruct(obj): # pylint: disable=no-self-argument + def deconstruct(obj): # pylint: disable=no-self-argument path, args, kwargs = FileSystemStorage.deconstruct(obj) kwargs["location"] = None - return (path, args, kwargs) + return path, args, kwargs + + +class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage): + """FileSystemStorage that shadows writes to the blob store as well""" + + def __init__( + self, + kind: str, + location=None, + base_url=None, + file_permissions_mode=None, + directory_permissions_mode=None, + ): + self.kind = kind + super().__init__( + location, base_url, file_permissions_mode, directory_permissions_mode + ) + + def save(self, name, content, max_length=None): + # Write content to the filesystem - this deals with chunks, etc... + saved_name = super().save(name, content, max_length) + + # Retrieve the content and write to the blob store + try: + with self.open(saved_name, "rb") as f: + store_file(self.kind, saved_name, f, allow_overwrite=True) + except Exception as err: + log(f"Failed to shadow {saved_name} at {self.kind}:{saved_name}: {err}") + return saved_name From 4b170aa0dd5dfe39715e3c58d569b4dc5d3e7bc1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 21:50:08 -0400 Subject: [PATCH 37/87] feat: shadow floorplans / host logos to the blob --- ietf/meeting/models.py | 14 +++++++++++--- ietf/settings.py | 4 +++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index a0c096464a..1c22d2bd38 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -39,7 +39,7 @@ from ietf.person.models import Person from ietf.utils.decorators import memoize from ietf.utils.history import find_history_replacements_active_at, find_history_active_at -from ietf.utils.storage import NoLocationMigrationFileSystemStorage +from ietf.utils.storage import BlobShadowFileSystemStorage, NoLocationMigrationFileSystemStorage from ietf.utils.text import xslugify from ietf.utils.timezone import datetime_from_date, date_today from ietf.utils.models import ForeignKey @@ -527,7 +527,12 @@ class FloorPlan(models.Model): modified= models.DateTimeField(auto_now=True) meeting = ForeignKey(Meeting) order = models.SmallIntegerField() - image = models.ImageField(storage=NoLocationMigrationFileSystemStorage(), upload_to=floorplan_path, blank=True, default=None) + image = models.ImageField( + storage=BlobShadowFileSystemStorage(kind="floorplan"), + upload_to=floorplan_path, + blank=True, + default=None, + ) # class Meta: ordering = ['-id',] @@ -1433,7 +1438,10 @@ class MeetingHost(models.Model): name = models.CharField(max_length=255, blank=False) # TODO-BLOBSTORE - capture these logos and look for other ImageField like model fields. logo = MissingOkImageField( - storage=NoLocationMigrationFileSystemStorage(location=settings.MEETINGHOST_LOGO_PATH), + storage=BlobShadowFileSystemStorage( + kind="meetinghostlogo", + location=settings.MEETINGHOST_LOGO_PATH, + ), upload_to=_host_upload_path, width_field='logo_width', height_field='logo_height', diff --git a/ietf/settings.py b/ietf/settings.py index a015850909..898078ef51 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -761,7 +761,9 @@ def skip_unreadable_post(record): "chatlog", "polls", "staging", - "bibxml-ids" + "bibxml-ids", + "floorplan", + "meetinghostlogo", ] # Override this in settings_local.py if needed From a6bd585e577614303718c3668170d4b61c8f5304 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 21:53:43 -0400 Subject: [PATCH 38/87] chore: remove unused import --- ietf/meeting/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 1c22d2bd38..5284420731 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -39,7 +39,7 @@ from ietf.person.models import Person from ietf.utils.decorators import memoize from ietf.utils.history import find_history_replacements_active_at, find_history_active_at -from ietf.utils.storage import BlobShadowFileSystemStorage, NoLocationMigrationFileSystemStorage +from ietf.utils.storage import BlobShadowFileSystemStorage from ietf.utils.text import xslugify from ietf.utils.timezone import datetime_from_date, date_today from ietf.utils.models import ForeignKey From 53bfb80ee814152afd9131ef71d1ba838d8422dd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 22:19:18 -0400 Subject: [PATCH 39/87] feat: strip path from blob shadow names --- ietf/utils/storage.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py index 148224c3c6..06c95552f5 100644 --- a/ietf/utils/storage.py +++ b/ietf/utils/storage.py @@ -1,5 +1,7 @@ # Copyright The IETF Trust 2020-2025, All Rights Reserved """Django Storage classes""" +from pathlib import Path + from django.core.files.storage import FileSystemStorage from ietf.doc.storage_utils import store_file from .log import log @@ -14,7 +16,10 @@ def deconstruct(obj): # pylint: disable=no-self-argument class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage): - """FileSystemStorage that shadows writes to the blob store as well""" + """FileSystemStorage that shadows writes to the blob store as well + + Strips directories from the filename when naming the blob. + """ def __init__( self, @@ -34,9 +39,10 @@ def save(self, name, content, max_length=None): saved_name = super().save(name, content, max_length) # Retrieve the content and write to the blob store + blob_name = Path(saved_name).name # strips path try: with self.open(saved_name, "rb") as f: - store_file(self.kind, saved_name, f, allow_overwrite=True) + store_file(self.kind, blob_name, f, allow_overwrite=True) except Exception as err: - log(f"Failed to shadow {saved_name} at {self.kind}:{saved_name}: {err}") - return saved_name + log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}") + return saved_name # includes the path! From 5a9f18a00d3fc2edc2d4a1e7a460725fd7600dbb Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 22:34:02 -0400 Subject: [PATCH 40/87] feat: shadow photos / thumbs --- ietf/person/models.py | 16 +++++++++++++--- ietf/settings.py | 2 ++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ietf/person/models.py b/ietf/person/models.py index 85989acfc1..05d5a361a7 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -29,7 +29,7 @@ from ietf.name.models import ExtResourceName from ietf.person.name import name_parts, initials, plain_name from ietf.utils.mail import send_mail_preformatted -from ietf.utils.storage import NoLocationMigrationFileSystemStorage +from ietf.utils.storage import BlobShadowFileSystemStorage from ietf.utils.mail import formataddr from ietf.person.name import unidecode_name from ietf.utils import log @@ -60,8 +60,18 @@ class Person(models.Model): pronouns_selectable = jsonfield.JSONCharField("Pronouns", max_length=120, blank=True, null=True, default=list ) pronouns_freetext = models.CharField(" ", max_length=30, null=True, blank=True, help_text="Optionally provide your personal pronouns. These will be displayed on your public profile page and alongside your name in Meetecho and, in future, other systems. Select any number of the checkboxes OR provide a custom string up to 30 characters.") biography = models.TextField(blank=True, help_text="Short biography for use on leadership pages. Use plain text or reStructuredText markup.") - photo = models.ImageField(storage=NoLocationMigrationFileSystemStorage(), upload_to=settings.PHOTOS_DIRNAME, blank=True, default=None) - photo_thumb = models.ImageField(storage=NoLocationMigrationFileSystemStorage(), upload_to=settings.PHOTOS_DIRNAME, blank=True, default=None) + photo = models.ImageField( + storage=BlobShadowFileSystemStorage("photo"), + upload_to=settings.PHOTOS_DIRNAME, + blank=True, + default=None, + ) + photo_thumb = models.ImageField( + storage=BlobShadowFileSystemStorage("photothumb"), + upload_to=settings.PHOTOS_DIRNAME, + blank=True, + default=None, + ) name_from_draft = models.CharField("Full Name (from submission)", null=True, max_length=255, editable=False, help_text="Name as found in an Internet-Draft submission.") def __str__(self): diff --git a/ietf/settings.py b/ietf/settings.py index 10e2b2ed02..420c59e0ef 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -765,6 +765,8 @@ def skip_unreadable_post(record): "indexes", "floorplan", "meetinghostlogo", + "photo", + "photothumb", ] # Override this in settings_local.py if needed From d336ecc72ffbfe1fcdb39d51f484d2cc12f8940c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 22:50:08 -0400 Subject: [PATCH 41/87] refactor: combine photo and photothumb blob kinds The photos / thumbs were already dropped in the same directory, so let's not add a distinction at this point. --- ietf/person/models.py | 2 +- ietf/settings.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ietf/person/models.py b/ietf/person/models.py index 05d5a361a7..bcfaddca13 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -67,7 +67,7 @@ class Person(models.Model): default=None, ) photo_thumb = models.ImageField( - storage=BlobShadowFileSystemStorage("photothumb"), + storage=BlobShadowFileSystemStorage("photo"), upload_to=settings.PHOTOS_DIRNAME, blank=True, default=None, diff --git a/ietf/settings.py b/ietf/settings.py index 420c59e0ef..56b2f2953f 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -766,7 +766,6 @@ def skip_unreadable_post(record): "floorplan", "meetinghostlogo", "photo", - "photothumb", ] # Override this in settings_local.py if needed From 476d504c920c5dccd9e71c8998fdea35af501671 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 23:01:10 -0400 Subject: [PATCH 42/87] style: whitespace --- ietf/utils/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py index 06c95552f5..0ea7452ba0 100644 --- a/ietf/utils/storage.py +++ b/ietf/utils/storage.py @@ -17,7 +17,7 @@ def deconstruct(obj): # pylint: disable=no-self-argument class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage): """FileSystemStorage that shadows writes to the blob store as well - + Strips directories from the filename when naming the blob. """ From a4eef7fdfcd5e90e16b9a468cd48f93735aaa344 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 23:03:50 -0400 Subject: [PATCH 43/87] refactor: use kwargs consistently --- ietf/person/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/person/models.py b/ietf/person/models.py index bcfaddca13..93364478ae 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -61,13 +61,13 @@ class Person(models.Model): pronouns_freetext = models.CharField(" ", max_length=30, null=True, blank=True, help_text="Optionally provide your personal pronouns. These will be displayed on your public profile page and alongside your name in Meetecho and, in future, other systems. Select any number of the checkboxes OR provide a custom string up to 30 characters.") biography = models.TextField(blank=True, help_text="Short biography for use on leadership pages. Use plain text or reStructuredText markup.") photo = models.ImageField( - storage=BlobShadowFileSystemStorage("photo"), + storage=BlobShadowFileSystemStorage(kind="photo"), upload_to=settings.PHOTOS_DIRNAME, blank=True, default=None, ) photo_thumb = models.ImageField( - storage=BlobShadowFileSystemStorage("photo"), + storage=BlobShadowFileSystemStorage(kind="photo"), upload_to=settings.PHOTOS_DIRNAME, blank=True, default=None, From 1e994cd73412f8c060ff4b60c37533424fbc7882 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Jan 2025 23:06:32 -0400 Subject: [PATCH 44/87] chore: migrations --- ..._floorplan_image_alter_meetinghost_logo.py | 56 +++++++++++++++++++ ...r_person_photo_alter_person_photo_thumb.py | 38 +++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py create mode 100644 ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py diff --git a/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py new file mode 100644 index 0000000000..78d16dc0cf --- /dev/null +++ b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py @@ -0,0 +1,56 @@ +# Generated by Django 4.2.18 on 2025-01-29 03:05 + +from django.db import migrations, models +import ietf.meeting.models +import ietf.utils.fields +import ietf.utils.storage +import ietf.utils.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ("meeting", "0009_session_meetecho_recording_name"), + ] + + operations = [ + migrations.AlterField( + model_name="floorplan", + name="image", + field=models.ImageField( + blank=True, + default=None, + storage=ietf.utils.storage.BlobShadowFileSystemStorage( + kind="floorplan", location=None + ), + upload_to=ietf.meeting.models.floorplan_path, + ), + ), + migrations.AlterField( + model_name="meetinghost", + name="logo", + field=ietf.utils.fields.MissingOkImageField( + height_field="logo_height", + storage=ietf.utils.storage.BlobShadowFileSystemStorage( + kind="meetinghostlogo", location=None + ), + upload_to=ietf.meeting.models._host_upload_path, + validators=[ + ietf.utils.validators.MaxImageSizeValidator(400, 400), + ietf.utils.validators.WrappedValidator( + ietf.utils.validators.validate_file_size, True + ), + ietf.utils.validators.WrappedValidator( + ietf.utils.validators.validate_file_extension, + [".png", ".jpg", ".jpeg"], + ), + ietf.utils.validators.WrappedValidator( + ietf.utils.validators.validate_mime_type, + ["image/jpeg", "image/png"], + True, + ), + ], + width_field="logo_width", + ), + ), + ] diff --git a/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py new file mode 100644 index 0000000000..f2e6992d78 --- /dev/null +++ b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.18 on 2025-01-29 03:04 + +from django.db import migrations, models +import ietf.utils.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ("person", "0003_alter_personalapikey_endpoint"), + ] + + operations = [ + migrations.AlterField( + model_name="person", + name="photo", + field=models.ImageField( + blank=True, + default=None, + storage=ietf.utils.storage.BlobShadowFileSystemStorage( + kind="photo", location=None + ), + upload_to="photo", + ), + ), + migrations.AlterField( + model_name="person", + name="photo_thumb", + field=models.ImageField( + blank=True, + default=None, + storage=ietf.utils.storage.BlobShadowFileSystemStorage( + kind="photo", location=None + ), + upload_to="photo", + ), + ), + ] From 7ce4a3c0c181e543a1fd1fa622ddff5314d85e49 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 29 Jan 2025 10:50:30 -0400 Subject: [PATCH 45/87] refactor: better deconstruct(); rebuild migrations --- ...0_alter_floorplan_image_alter_meetinghost_logo.py | 6 +++--- ...04_alter_person_photo_alter_person_photo_thumb.py | 6 +++--- ietf/utils/storage.py | 12 +++++++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py index 78d16dc0cf..7d9a92b12d 100644 --- a/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py +++ b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.18 on 2025-01-29 03:05 +# Generated by Django 4.2.18 on 2025-01-29 14:49 from django.db import migrations, models import ietf.meeting.models @@ -21,7 +21,7 @@ class Migration(migrations.Migration): blank=True, default=None, storage=ietf.utils.storage.BlobShadowFileSystemStorage( - kind="floorplan", location=None + kind="", location=None ), upload_to=ietf.meeting.models.floorplan_path, ), @@ -32,7 +32,7 @@ class Migration(migrations.Migration): field=ietf.utils.fields.MissingOkImageField( height_field="logo_height", storage=ietf.utils.storage.BlobShadowFileSystemStorage( - kind="meetinghostlogo", location=None + kind="", location=None ), upload_to=ietf.meeting.models._host_upload_path, validators=[ diff --git a/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py index f2e6992d78..63c535af07 100644 --- a/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py +++ b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.18 on 2025-01-29 03:04 +# Generated by Django 4.2.18 on 2025-01-29 14:49 from django.db import migrations, models import ietf.utils.storage @@ -18,7 +18,7 @@ class Migration(migrations.Migration): blank=True, default=None, storage=ietf.utils.storage.BlobShadowFileSystemStorage( - kind="photo", location=None + kind="", location=None ), upload_to="photo", ), @@ -30,7 +30,7 @@ class Migration(migrations.Migration): blank=True, default=None, storage=ietf.utils.storage.BlobShadowFileSystemStorage( - kind="photo", location=None + kind="", location=None ), upload_to="photo", ), diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py index 0ea7452ba0..bd8324232c 100644 --- a/ietf/utils/storage.py +++ b/ietf/utils/storage.py @@ -9,9 +9,9 @@ class NoLocationMigrationFileSystemStorage(FileSystemStorage): - def deconstruct(obj): # pylint: disable=no-self-argument - path, args, kwargs = FileSystemStorage.deconstruct(obj) - kwargs["location"] = None + def deconstruct(self): + path, args, kwargs = super().deconstruct() + kwargs["location"] = None # don't record location in migrations return path, args, kwargs @@ -23,6 +23,7 @@ class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage): def __init__( self, + *, # disallow positional arguments kind: str, location=None, base_url=None, @@ -46,3 +47,8 @@ def save(self, name, content, max_length=None): except Exception as err: log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}") return saved_name # includes the path! + + def deconstruct(self): + path, args, kwargs = super().deconstruct() + kwargs["kind"] = "" # don't record "kind" in migrations + return path, args, kwargs From dae85e8f513b966de82095607b9eaf575e6c9d50 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 29 Jan 2025 13:15:15 -0600 Subject: [PATCH 46/87] fix: use new class in mack patch --- ietf/meeting/tests_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 66df1d06b9..85ec0bf212 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -112,7 +112,7 @@ def setUp(self): # files will upload to the locations specified in settings.py. # Note that this will affect any use of the storage class in # meeting.models - i.e., FloorPlan.image and MeetingHost.logo - self.patcher = patch('ietf.meeting.models.NoLocationMigrationFileSystemStorage.base_location', + self.patcher = patch('ietf.meeting.models.BlobShadowFileSystemStorage.base_location', new_callable=PropertyMock) mocked = self.patcher.start() mocked.return_value = self.storage_dir From e6343ac23b5fcb5b82ae115b4ef4c5f10e848788 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 29 Jan 2025 15:16:51 -0400 Subject: [PATCH 47/87] chore: add TODO --- ietf/nomcom/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/nomcom/models.py b/ietf/nomcom/models.py index 2ed1124c5c..c206e467bd 100644 --- a/ietf/nomcom/models.py +++ b/ietf/nomcom/models.py @@ -42,6 +42,7 @@ class ReminderDates(models.Model): class NomCom(models.Model): + # TODO-BLOBSTORE: migrate this to a database field instead of a FileField and update code accordingly public_key = models.FileField(storage=NoLocationMigrationFileSystemStorage(location=settings.NOMCOM_PUBLIC_KEYS_DIR), upload_to=upload_path_handler, blank=True, null=True) From 7a7fe267c2db62f6a75ddda34d8760afa2ffd6a6 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 29 Jan 2025 16:38:19 -0600 Subject: [PATCH 48/87] feat: store group index documents --- ietf/group/tasks.py | 11 +++++++++++ ietf/group/tests_info.py | 35 +++++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py index 8b4c994ba1..052e89132f 100644 --- a/ietf/group/tasks.py +++ b/ietf/group/tasks.py @@ -10,6 +10,7 @@ from django.conf import settings from django.template.loader import render_to_string +from ietf.doc.storage_utils import store_file from ietf.utils import log from .models import Group @@ -43,6 +44,11 @@ def generate_wg_charters_files_task(): encoding="utf8", ) + with charters_file.open() as f: + store_file("indexes", "1wg-charters.txt", f) + with charters_by_acronym_file.open() as f: + store_file("indexes", "1wg-charters-by-acronym.txt", f) + charter_copy_dests = [ getattr(settings, "CHARTER_COPY_PATH", None), getattr(settings, "CHARTER_COPY_OTHER_PATH", None), @@ -102,3 +108,8 @@ def generate_wg_summary_files_task(): ), encoding="utf8", ) + + with summary_file.open() as f: + store_file("indexes", "1wg-summary.txt", f) + with summary_by_acronym_file.open() as f: + store_file("indexes", "1wg-summary-by-acronym.txt", f) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 32d919c779..aaf937ee43 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -29,6 +29,7 @@ from ietf.community.utils import reset_name_contains_index_for_rule from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State +from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group from ietf.group.admin import GroupForm as AdminGroupForm from ietf.group.factories import (GroupFactory, RoleFactory, GroupEventFactory, @@ -303,20 +304,26 @@ def test_generate_wg_summary_files_task(self): generate_wg_summary_files_task() - summary_by_area_contents = ( - Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt" - ).read_text(encoding="utf8") - self.assertIn(group.parent.name, summary_by_area_contents) - self.assertIn(group.acronym, summary_by_area_contents) - self.assertIn(group.name, summary_by_area_contents) - self.assertIn(chair.address, summary_by_area_contents) - - summary_by_acronym_contents = ( - Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt" - ).read_text(encoding="utf8") - self.assertIn(group.acronym, summary_by_acronym_contents) - self.assertIn(group.name, summary_by_acronym_contents) - self.assertIn(chair.address, summary_by_acronym_contents) + for summary_by_area_contents in [ + ( + Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt" + ).read_text(encoding="utf8"), + retrieve_str("indexes", "1wg-summary.txt") + ]: + self.assertIn(group.parent.name, summary_by_area_contents) + self.assertIn(group.acronym, summary_by_area_contents) + self.assertIn(group.name, summary_by_area_contents) + self.assertIn(chair.address, summary_by_area_contents) + + for summary_by_acronym_contents in [ + ( + Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt" + ).read_text(encoding="utf8"), + retrieve_str("indexes", "1wg-summary-by-acronym.txt") + ]: + self.assertIn(group.acronym, summary_by_acronym_contents) + self.assertIn(group.name, summary_by_acronym_contents) + self.assertIn(chair.address, summary_by_acronym_contents) def test_chartering_groups(self): group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group From b0fba2a85bd196197ec65419c875becdc91c1d9c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 29 Jan 2025 16:55:20 -0600 Subject: [PATCH 49/87] chore: identify more TODO --- ietf/doc/tasks.py | 4 ++-- ietf/doc/utils.py | 2 +- ietf/doc/views_review.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 6eb901e6c7..e24c58e1e7 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -84,7 +84,7 @@ def generate_idnits2_rfc_status_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" blob = generate_idnits2_rfc_status() try: - outpath.write_text(blob, encoding="utf8") + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfc-status: {e}") @@ -94,7 +94,7 @@ def generate_idnits2_rfcs_obsoleted_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" blob = generate_idnits2_rfcs_obsoleted() try: - outpath.write_text(blob, encoding="utf8") + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 10fe9ff2d7..3ddd904c75 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1510,7 +1510,7 @@ def update_or_create_draft_bibxml_file(doc, rev): existing_bibxml = "" if normalized_bibxml.strip() != existing_bibxml.strip(): log.log(f"Writing {ref_rev_file_path}") - ref_rev_file_path.write_text(normalized_bibxml, encoding="utf8") + ref_rev_file_path.write_text(normalized_bibxml, encoding="utf8") # TODO-BLOBSTORE def ensure_draft_bibxml_path_exists(): diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index bb9e56742d..570072c8b7 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -804,7 +804,7 @@ def complete_review(request, name, assignment_id=None, acronym=None): content = form.cleaned_data['review_content'] review_path = Path(review.get_file_path()) / f"{review.name}.txt" - review_path.write_text(content) + review_path.write_text(content) # TODO-BLOBSTORE review_ftp_path = Path(settings.FTP_DIR) / "review" / review_path.name # See https://github.com/ietf-tools/datatracker/issues/6941 - when that's # addressed, making this link should not be conditional From 79a62e6880d3d896aa2a884252678a6caf2af5ea Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 30 Jan 2025 15:35:55 -0600 Subject: [PATCH 50/87] feat: store reviews --- ietf/doc/tests_review.py | 5 +++++ ietf/doc/views_review.py | 4 +++- ietf/settings.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index a956fd3287..e93bc02181 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -20,6 +20,7 @@ import debug # pyflakes:ignore +from ietf.doc.storage_utils import retrieve_str import ietf.review.mailarch from ietf.doc.factories import ( NewRevisionDocEventFactory, IndividualDraftFactory, WgDraftFactory, @@ -63,6 +64,10 @@ def verify_review_files_were_written(self, assignment, expected_content = "This review_file = Path(self.review_subdir) / f"{assignment.review.name}.txt" content = review_file.read_text() self.assertEqual(content, expected_content) + self.assertEqual( + retrieve_str("review", review_file.name), + expected_content + ) review_ftp_file = Path(settings.FTP_DIR) / "review" / review_file.name self.assertTrue(review_file.samefile(review_ftp_file)) diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 570072c8b7..6c5010d139 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -30,6 +30,7 @@ from ietf.doc.models import (Document, NewRevisionDocEvent, State, LastCallDocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent, DocumentAuthor) +from ietf.doc.storage_utils import store_str from ietf.name.models import (ReviewRequestStateName, ReviewAssignmentStateName, ReviewResultName, ReviewTypeName) from ietf.person.models import Person @@ -804,7 +805,8 @@ def complete_review(request, name, assignment_id=None, acronym=None): content = form.cleaned_data['review_content'] review_path = Path(review.get_file_path()) / f"{review.name}.txt" - review_path.write_text(content) # TODO-BLOBSTORE + review_path.write_text(content) + store_str("review", f"{review.name}.txt", content) review_ftp_path = Path(settings.FTP_DIR) / "review" / review_path.name # See https://github.com/ietf-tools/datatracker/issues/6941 - when that's # addressed, making this link should not be conditional diff --git a/ietf/settings.py b/ietf/settings.py index 56b2f2953f..fab2b944e8 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -766,6 +766,7 @@ def skip_unreadable_post(record): "floorplan", "meetinghostlogo", "photo", + "review", ] # Override this in settings_local.py if needed From e22bac79621d84ef69394323e3dea47f6c5ac4b4 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 31 Jan 2025 12:14:58 -0600 Subject: [PATCH 51/87] fix: repair merge --- ietf/meeting/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index a5b5aeffbb..ee914724a0 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -776,7 +776,7 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120]) # Whole file sanitization; add back what's missing from a complete # document (sanitize will remove these). - clean = sanitize_document(text) + clean = clean_html(text) clean_bytes = clean.encode('utf8') destination.write(clean_bytes) # Assumes contents of subdir are always document type ids From 2effb954c0721995bea7fd7f29b66b8c4813fe73 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 31 Jan 2025 12:21:32 -0600 Subject: [PATCH 52/87] chore: remove unnecessary TODO --- ietf/meeting/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index ee914724a0..dc9fb6457c 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -226,8 +226,6 @@ def generate_bluesheet(request, session): 'session': session, 'data': data, }) - # TODO-BLOBSTORE Verify that this is only creating a file-like object to pass along - # if so, we can do this in memory and not involve disk. fd, name = tempfile.mkstemp(suffix=".txt", text=True) os.close(fd) with open(name, "w") as file: From 78c94a6f89e0b254a37d5ecab537f10122b3648f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 6 Feb 2025 11:55:56 -0600 Subject: [PATCH 53/87] feat: StoredObject metadata --- dev/deploy-to-container/settings_local.py | 2 +- dev/diff/settings_local.py | 2 +- dev/tests/settings_local.py | 2 +- docker/configs/settings_local.py | 2 +- ietf/doc/admin.py | 8 +- ietf/doc/expire.py | 2 +- ...ject_storedobject_unique_name_per_store.py | 66 +++++++++ ietf/doc/models.py | 52 ++++++- ietf/doc/resources.py | 25 +++- ietf/doc/storage_backends.py | 137 ++++++++++++++++++ ietf/doc/storage_utils.py | 79 +++++----- ietf/doc/tests_draft.py | 4 +- ietf/doc/views_bofreq.py | 5 +- ietf/doc/views_charter.py | 3 +- ietf/doc/views_conflict_review.py | 3 +- ietf/doc/views_material.py | 3 +- ietf/doc/views_review.py | 3 +- ietf/doc/views_statement.py | 9 +- ietf/doc/views_status_change.py | 3 +- ietf/group/tasks.py | 16 +- ietf/idindex/tasks.py | 2 +- ietf/liaisons/forms.py | 3 +- ietf/meeting/forms.py | 3 +- ietf/meeting/tests_views.py | 9 +- ietf/meeting/utils.py | 4 + ietf/meeting/views.py | 4 +- ietf/settings_test.py | 2 +- ietf/submit/tests.py | 47 +++++- ietf/submit/utils.py | 15 +- ietf/utils/test_runner.py | 73 ++++++---- 30 files changed, 452 insertions(+), 136 deletions(-) create mode 100644 ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py create mode 100644 ietf/doc/storage_backends.py diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 2908a1d97c..df0e7cc7f1 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -83,7 +83,7 @@ for storagename in MORE_STORAGE_NAMES: STORAGES[storagename] = { - "BACKEND": "storages.backends.s3.S3Storage", + "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", "OPTIONS": dict( endpoint_url="http://blobstore:9000", access_key="minio_root", diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index 785c6b5648..b0994dcfbf 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -70,7 +70,7 @@ for storagename in MORE_STORAGE_NAMES: STORAGES[storagename] = { - "BACKEND": "storages.backends.s3.S3Storage", + "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", "OPTIONS": dict( endpoint_url="http://blobstore:9000", access_key="minio_root", diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index 42ab6a019a..b8815b837b 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -69,7 +69,7 @@ for storagename in MORE_STORAGE_NAMES: STORAGES[storagename] = { - "BACKEND": "storages.backends.s3.S3Storage", + "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", "OPTIONS": dict( endpoint_url="http://blobstore:9000", access_key="minio_root", diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 850cdd017d..8771053419 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -40,7 +40,7 @@ # ] for storagename in MORE_STORAGE_NAMES: STORAGES[storagename] = { - "BACKEND": "storages.backends.s3.S3Storage", + "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", "OPTIONS": dict( endpoint_url="http://blobstore:9000", access_key="minio_root", diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 301d32d7cc..db3b24b2d2 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -12,7 +12,7 @@ TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent, AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL, ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent ) + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject ) from ietf.utils.validators import validate_external_resource_value @@ -218,3 +218,9 @@ class DocExtResourceAdmin(admin.ModelAdmin): search_fields = ['doc__name', 'value', 'display_name', 'name__slug',] raw_id_fields = ['doc', ] admin.site.register(DocExtResource, DocExtResourceAdmin) + +class StoredObjectAdmin(admin.ModelAdmin): + list_display = ['store', 'name', 'modified', 'deleted'] + list_filter = ['deleted'] + search_fields = ['store', 'name', 'doc_name', 'doc_rev', 'deleted'] +admin.site.register(StoredObject, StoredObjectAdmin) diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index c52e0679aa..bf8523aa98 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -160,7 +160,7 @@ def remove_ftp_copy(f): def remove_from_active_draft_storage(file): # Assumes the glob will never find a file with no suffix ext = file.suffix[1:] - remove_from_storage("active-draft", f"{ext}/{file.name}") + remove_from_storage("active-draft", f"{ext}/{file.name}", warn_if_missing=False) # Note that the object is already in the "draft" storage. src_dir = Path(settings.INTERNET_DRAFT_PATH) diff --git a/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py b/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py new file mode 100644 index 0000000000..9161007d4e --- /dev/null +++ b/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py @@ -0,0 +1,66 @@ +# Generated by Django 4.2.18 on 2025-02-04 20:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("doc", "0024_remove_ad_is_watching_states"), + ] + + operations = [ + migrations.CreateModel( + name="StoredObject", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("store", models.CharField(max_length=256)), + ("name", models.CharField(max_length=1024)), + ("sha384", models.CharField(max_length=96)), + ("len", models.PositiveBigIntegerField()), + ( + "store_created", + models.DateTimeField( + help_text="The instant the object ws first placed in the store" + ), + ), + ( + "created", + models.DateTimeField( + help_text="Instant object became known. May not be the same as the storage's created value for the instance. It will hold ctime for objects imported from older disk storage" + ), + ), + ( + "modified", + models.DateTimeField( + help_text="Last instant object was modified. May not be the same as the storage's modified value for the instance. It will hold mtime for objects imported from older disk storage unless they've actually been overwritten more recently" + ), + ), + ("doc_name", models.CharField(blank=True, max_length=255, null=True)), + ("doc_rev", models.CharField(blank=True, max_length=16, null=True)), + ("deleted", models.DateTimeField(null=True)), + ], + options={ + "indexes": [ + models.Index( + fields=["doc_name", "doc_rev"], + name="doc_storedo_doc_nam_d04465_idx", + ) + ], + }, + ), + migrations.AddConstraint( + model_name="storedobject", + constraint=models.UniqueConstraint( + fields=("store", "name"), name="unique_name_per_store" + ), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 03698c80c3..840eafd38a 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -9,14 +9,16 @@ import django.db import rfc2html +from io import BufferedReader from pathlib import Path from lxml import etree -from typing import Optional, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING, Union from weasyprint import HTML as wpHTML from weasyprint.text.fonts import FontConfiguration from django.db import models from django.core import checks +from django.core.files.base import File from django.core.cache import caches from django.core.validators import URLValidator, RegexValidator from django.urls import reverse as urlreverse @@ -30,6 +32,11 @@ import debug # pyflakes:ignore from ietf.group.models import Group +from ietf.doc.storage_utils import ( + store_str as utils_store_str, + store_bytes as utils_store_bytes, + store_file as utils_store_file +) from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName, DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName, DocUrlTagName, ExtResourceName) @@ -714,6 +721,21 @@ def referenced_by_rfcs_as_rfc_or_draft(self): if self.type_id == "rfc" and self.came_from_draft(): refs_to |= self.came_from_draft().referenced_by_rfcs() return refs_to + + def store_str( + self, name: str, content: str, allow_overwrite: bool = False + ) -> None: + return utils_store_str(self.type_id, name, content, allow_overwrite, self.name, self.rev) + + def store_bytes( + self, name: str, content: bytes, allow_overwrite: bool = False, doc_name: Optional[str] = None, doc_rev: Optional[str] = None + ) -> None: + return utils_store_bytes(self.type_id, name, content, allow_overwrite, self.name, self.rev) + + def store_file( + self, name: str, file: Union[File,BufferedReader], allow_overwrite: bool = False, doc_name: Optional[str] = None, doc_rev: Optional[str] = None + ) -> None: + return utils_store_file(self.type_id, name, file, allow_overwrite, self.name, self.rev) class Meta: abstract = True @@ -1538,3 +1560,31 @@ class BofreqEditorDocEvent(DocEvent): class BofreqResponsibleDocEvent(DocEvent): """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """ responsible = models.ManyToManyField('person.Person', blank=True) + +class StoredObject(models.Model): + """Hold metadata about objects placed in object storage""" + + store = models.CharField(max_length=256) + name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3 + sha384 = models.CharField(max_length=96) + len = models.PositiveBigIntegerField() + store_created = models.DateTimeField(help_text="The instant the object ws first placed in the store") + created = models.DateTimeField( + null=False, + help_text="Instant object became known. May not be the same as the storage's created value for the instance. It will hold ctime for objects imported from older disk storage" + ) + modified = models.DateTimeField( + null=False, + help_text="Last instant object was modified. May not be the same as the storage's modified value for the instance. It will hold mtime for objects imported from older disk storage unless they've actually been overwritten more recently" + ) + doc_name = models.CharField(max_length=255, null=True, blank=True) + doc_rev = models.CharField(max_length=16, null=True, blank=True) + deleted = models.DateTimeField(null=True) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['store', 'name'], name='unique_name_per_store'), + ] + indexes = [ + models.Index(fields=["doc_name", "doc_rev"]), + ] diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index bba57013b9..157a3ad556 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -18,7 +18,7 @@ RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent,BofreqResponsibleDocEvent) + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject) from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource class BallotTypeResource(ModelResource): @@ -842,3 +842,26 @@ class Meta: "responsible": ALL_WITH_RELATIONS, } api.doc.register(BofreqResponsibleDocEventResource()) + + +class StoredObjectResource(ModelResource): + class Meta: + queryset = StoredObject.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'storedobject' + ordering = ['id', ] + filtering = { + "id": ALL, + "store": ALL, + "name": ALL, + "sha384": ALL, + "len": ALL, + "store_created": ALL, + "created": ALL, + "modified": ALL, + "doc_name": ALL, + "doc_rev": ALL, + "deleted": ALL, + } +api.doc.register(StoredObjectResource()) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py new file mode 100644 index 0000000000..07cd14ec80 --- /dev/null +++ b/ietf/doc/storage_backends.py @@ -0,0 +1,137 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import debug # pyflakes:ignore + +from hashlib import sha384 +from io import BufferedReader +from storages.backends.s3 import S3Storage +from storages.utils import is_seekable +from typing import Dict, Optional, Union + +from django.core.files.base import File + +from ietf.doc.models import StoredObject +from ietf.utils.log import log +from ietf.utils.timezone import timezone + + +class CustomS3Storage(S3Storage): + + def __init__(self, **settings): + self.in_flight_custom_metadata: Dict[str, Dict[str, str]] = {} + return super().__init__(**settings) + + def store_file( + self, + kind: str, + name: str, + file: Union[File, BufferedReader], + allow_overwrite: bool = False, + doc_name: Optional[str] = None, + doc_rev: Optional[str] = None, + ): + is_new = not self.exists_in_storage(kind, name) + # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"') + if not allow_overwrite and not is_new: + log(f"Failed to save {kind}:{name} - name already exists in store") + debug.show('f"Failed to save {kind}:{name} - name already exists in store"') + debug.traceback() + raise Exception("Not ignoring overwrite attempts while testing") + else: + try: + new_name = self.save(name, file) + now = timezone.now() + existing_record = StoredObject.objects.filter(store=kind, name=name) + if existing_record.exists(): + # Note this is updating a queryset which is guaranteed by constraints to have one object + existing_record.update( + sha384=self.in_flight_custom_metadata[name]["sha384"], + len=int(self.in_flight_custom_metadata[name]["len"]), + modified=now, + ) + else: + StoredObject.objects.create( + store=kind, + name=name, + sha384=self.in_flight_custom_metadata[name]["sha384"], + len=int(self.in_flight_custom_metadata[name]["len"]), + store_created=now, + created=now, + modified=now, + doc_name=doc_name, + doc_rev=doc_rev, + ) + if new_name != name: + complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." + log(complaint) + debug.show("complaint") + # Note that we are otherwise ignoring this condition - it should become an error later. + except Exception as e: + # Log and then swallow the exception while we're learning. + # Don't let failure pass so quietly when these are the autoritative bits. + log(f"Failed to save {kind}:{name}", e) + raise e + debug.show("type(e)") + debug.show("e") + debug.traceback() + finally: + del self.in_flight_custom_metadata[name] + return None + + def exists_in_storage(self, kind: str, name: str) -> bool: + try: + # open is realized with a HEAD + # See https://github.com/jschneier/django-storages/blob/b79ea310201e7afd659fe47e2882fe59aae5b517/storages/backends/s3.py#L528 + with self.open(name): + return True + except FileNotFoundError: + return False + + def remove_from_storage( + self, kind: str, name: str, warn_if_missing: bool = True + ) -> None: + now = timezone.now() + try: + with self.open(name): + pass + self.delete(name) + # debug.show('f"deleted {name} from {kind} storage"') + except FileNotFoundError: + if warn_if_missing: + complaint = ( + f"WARNING: Asked to delete non-existant {name} from {kind} storage" + ) + log(complaint) + debug.show("complaint") + existing_record = StoredObject.objects.filter(store=kind, name=name) + if not existing_record.exists() and warn_if_missing: + complaint = f"WARNING: Asked to delete {name} from {kind} storage, but there was no matching StorageObject" + log(complaint) + debug.show("complaint") + else: + # Note that existing_record is a queryset that will have one matching object + existing_record.update(deleted=now) + + def _get_write_parameters(self, name, content=None): + # debug.show('f"getting write parameters for {name}"') + params = super()._get_write_parameters(name, content) + if "Metadata" not in params: + params["Metadata"] = {} + if not is_seekable(content): + # TODO-BLOBSTORE + debug.say("Encountered Non-Seekable content") + raise NotImplementedError("cannot handle unseekable content") + content.seek(0) + content_bytes = content.read() + if not isinstance( + content_bytes, bytes + ): # TODO-BLOBSTORE: This is sketch-development only -remove before committing + raise Exception(f"Expected bytes - got {type(content_bytes)}") + content.seek(0) + metadata = { + "len": f"{len(content_bytes)}", + "sha384": f"{sha384(content_bytes).hexdigest()}", + } + params["Metadata"].update(metadata) + self.in_flight_custom_metadata[name] = metadata + return params diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 82f0e3bcb6..82cf6354e3 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -1,18 +1,19 @@ # Copyright The IETF Trust 2025, All Rights Reserved from io import BufferedReader -from typing import Union +from typing import Optional, Union import debug # pyflakes ignore from django.conf import settings from django.core.files.base import ContentFile, File -from django.core.files.storage import storages, Storage +from django.core.files.storage import storages -from ietf.utils.log import log +# TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type +def _get_storage(kind: str): -def _get_storage(kind: str) -> Storage: if kind in settings.MORE_STORAGE_NAMES: + # TODO-BLOBSTORE - add a checker that verifies configuration will only return CustomS3Storages return storages[kind] else: debug.say(f"Got into not-implemented looking for {kind}") @@ -21,58 +22,48 @@ def _get_storage(kind: str) -> Storage: def exists_in_storage(kind: str, name: str) -> bool: store = _get_storage(kind) - try: - # open is realized with a HEAD - # See https://github.com/jschneier/django-storages/blob/b79ea310201e7afd659fe47e2882fe59aae5b517/storages/backends/s3.py#L528 - with store.open(name): - return True - except FileNotFoundError: - return False + return store.exists_in_storage(kind, name) -def remove_from_storage(kind: str, name: str) -> None: +def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None: store = _get_storage(kind) - try: - with store.open(name): - pass - store.delete(name) - # debug.show('f"deleted {name} from {kind} storage"') - except FileNotFoundError: - complaint = f"WARNING: Asked to delete non-existant {name} from {kind} storage" - log(complaint) - # debug.show("complaint") - - -def store_file(kind: str, name: str, file: Union[File,BufferedReader], allow_overwrite: bool = False) -> None: + store.remove_from_storage(kind, name, warn_if_missing) + return None + + +# TODO-BLOBSTORE: Try to refactor `kind` out of the signature of the methods already on the custom store (which knows its kind) +def store_file( + kind: str, + name: str, + file: Union[File, BufferedReader], + allow_overwrite: bool = False, + doc_name: Optional[str] = None, + doc_rev: Optional[str] = None, +) -> None: # debug.show('f"asked to store {name} into {kind}"') store = _get_storage(kind) - if not allow_overwrite and store.exists(name): - log(f"Failed to save {kind}:{name} - name already exists in store") - debug.show('f"Failed to save {kind}:{name} - name already exists in store"') - else: - try: - new_name = store.save(name, file) - except Exception as e: - # Log and then swallow the exception while we're learning. - # Don't let failure pass so quietly when these are the autoritative bits. - log(f"Failed to save {kind}:{name}", e) - debug.show("e") - return None - if new_name != name: - complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." - log(complaint) - debug.show("complaint") - return None - # TODO return value on other paths + store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev) + return None + def store_bytes( - kind: str, name: str, content: bytes, allow_overwrite: bool = False + kind: str, + name: str, + content: bytes, + allow_overwrite: bool = False, + doc_name: Optional[str] = None, + doc_rev: Optional[str] = None, ) -> None: return store_file(kind, name, ContentFile(content), allow_overwrite) def store_str( - kind: str, name: str, content: str, allow_overwrite: bool = False + kind: str, + name: str, + content: str, + allow_overwrite: bool = False, + doc_name: Optional[str] = None, + doc_rev: Optional[str] = None, ) -> None: content_bytes = content.encode("utf-8") return store_bytes(kind, name, content_bytes, allow_overwrite) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index d0a45f3723..4753c4ff0c 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -581,8 +581,8 @@ def write_draft_file(self, name, size): _, ext = os.path.splitext(name) if ext: ext=ext[1:] - store_str("active-draft", f"{ext}/{name}", "a"*size) - store_str("draft", f"{ext}/{name}", "a"*size) + store_str("active-draft", f"{ext}/{name}", "a"*size, allow_overwrite=True) + store_str("draft", f"{ext}/{name}", "a"*size, allow_overwrite=True) class ResurrectTests(DraftFileMixin, TestCase): diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index e52dd2c56f..71cbe30491 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -17,7 +17,6 @@ email_bofreq_new_revision, email_bofreq_responsible_changed) from ietf.doc.models import (Document, DocEvent, NewRevisionDocEvent, BofreqEditorDocEvent, BofreqResponsibleDocEvent, State) -from ietf.doc.storage_utils import store_str from ietf.doc.utils import add_state_change_event from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible from ietf.ietfauth.utils import has_role, role_required @@ -102,7 +101,7 @@ def submit(request, name): content = form.cleaned_data['bofreq_content'] with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination: destination.write(content) - store_str("bofreq", bofreq.get_base_name(), content) + bofreq.store_str(bofreq.get_base_name(), content) email_bofreq_new_revision(request, bofreq) return redirect('ietf.doc.views_doc.document_main', name=bofreq.name) @@ -177,7 +176,7 @@ def new_bof_request(request): content = form.cleaned_data['bofreq_content'] with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination: destination.write(content) - store_str("bofreq", bofreq.get_base_name(), content) + bofreq.store_str(bofreq.get_base_name(), content) email_bofreq_new_revision(request, bofreq) return redirect('ietf.doc.views_doc.document_main', name=bofreq.name) diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index 74a3c0124d..e899f59227 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -26,7 +26,6 @@ from ietf.doc.models import ( Document, DocHistory, State, DocEvent, BallotDocEvent, BallotPositionDocEvent, InitialReviewDocEvent, NewRevisionDocEvent, WriteupDocEvent, TelechatDocEvent ) -from ietf.doc.storage_utils import store_str from ietf.doc.utils import ( add_state_change_event, close_open_ballots, create_ballot, get_chartering_type ) from ietf.doc.utils_charter import ( historic_milestones_for_charter, @@ -457,7 +456,7 @@ def submit(request, name, option=None): "There was an error creating a hardlink at %s pointing to %s" % (ftp_filename, charter_filename) ) - store_str("charter", charter_filename.name, content) + charter.store_str(charter_filename.name, content) if option in ["initcharter", "recharter"] and charter.ad == None: diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py index eb26fff5e7..159f1340a4 100644 --- a/ietf/doc/views_conflict_review.py +++ b/ietf/doc/views_conflict_review.py @@ -19,7 +19,6 @@ from ietf.doc.models import ( BallotDocEvent, BallotPositionDocEvent, DocEvent, Document, NewRevisionDocEvent, State ) -from ietf.doc.storage_utils import store_str from ietf.doc.utils import ( add_state_change_event, close_open_ballots, create_ballot_if_not_open, update_telechat ) from ietf.doc.mails import email_iana, email_ad_approved_conflict_review @@ -199,7 +198,7 @@ def save(self, review): "There was an error creating a hardlink at %s pointing to %s: %s" % (ftp_filepath, filepath, e) ) - store_str("conflrev", basename, content) + review.store_str(basename, content) #This is very close to submit on charter - can we get better reuse? @role_required('Area Director','Secretariat') diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index afc833eb95..6f8b8a8f12 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -19,7 +19,6 @@ from ietf.doc.models import Document, DocTypeName, DocEvent, State from ietf.doc.models import NewRevisionDocEvent -from ietf.doc.storage_utils import store_file from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules from ietf.group.models import Group from ietf.group.utils import can_manage_materials @@ -169,7 +168,7 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): for chunk in f.chunks(): dest.write(chunk) f.seek(0) - store_file(doc.type_id, basename, f) + doc.store_file(basename, f) if not doc.meeting_related(): log.assertion('doc.type_id == "slides"') ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 6c5010d139..1f23c435fa 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -30,7 +30,6 @@ from ietf.doc.models import (Document, NewRevisionDocEvent, State, LastCallDocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent, DocumentAuthor) -from ietf.doc.storage_utils import store_str from ietf.name.models import (ReviewRequestStateName, ReviewAssignmentStateName, ReviewResultName, ReviewTypeName) from ietf.person.models import Person @@ -806,7 +805,7 @@ def complete_review(request, name, assignment_id=None, acronym=None): review_path = Path(review.get_file_path()) / f"{review.name}.txt" review_path.write_text(content) - store_str("review", f"{review.name}.txt", content) + review.store_str(f"{review.name}.txt", content, allow_overwrite=True) # We have a bug that review revisions dont create a new version! review_ftp_path = Path(settings.FTP_DIR) / "review" / review_path.name # See https://github.com/ietf-tools/datatracker/issues/6941 - when that's # addressed, making this link should not be conditional diff --git a/ietf/doc/views_statement.py b/ietf/doc/views_statement.py index 1af70601cc..9dc8c8ad69 100644 --- a/ietf/doc/views_statement.py +++ b/ietf/doc/views_statement.py @@ -10,7 +10,6 @@ from django.views.decorators.cache import cache_control from django.shortcuts import get_object_or_404, render, redirect from django.template.loader import render_to_string -from ietf.doc.storage_utils import store_file, store_str from ietf.utils import markdown from django.utils.html import escape @@ -142,10 +141,10 @@ def submit(request, name): for chunk in f.chunks(): destination.write(chunk) f.seek(0) - store_file("statement", statement.uploaded_filename, f) + statement.store_file(statement.uploaded_filename, f) else: destination.write(markdown_content) - store_str("statement", statement.uploaded_filename, markdown_content) + statement.store_str(statement.uploaded_filename, markdown_content) return redirect("ietf.doc.views_doc.document_main", name=statement.name) else: if statement.uploaded_filename.endswith("pdf"): @@ -262,10 +261,10 @@ def new_statement(request): for chunk in f.chunks(): destination.write(chunk) f.seek(0) - store_file("statement", statement.uploaded_filename, f) + statement.store_file(statement.uploaded_filename, f) else: destination.write(markdown_content) - store_str("statement", statement.uploaded_filename, markdown_content) + statement.store_str(statement.uploaded_filename, markdown_content) return redirect("ietf.doc.views_doc.document_main", name=statement.name) else: diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index 388a97d608..2bccc213c4 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -26,7 +26,6 @@ BallotPositionDocEvent, NewRevisionDocEvent, WriteupDocEvent, STATUSCHANGE_RELATIONS ) from ietf.doc.forms import AdForm from ietf.doc.lastcall import request_last_call -from ietf.doc.storage_utils import store_str from ietf.doc.utils import add_state_change_event, update_telechat, close_open_ballots, create_ballot_if_not_open from ietf.doc.views_ballot import LastCallTextForm from ietf.group.models import Group @@ -165,7 +164,7 @@ def save(self, doc): else: content = self.cleaned_data['content'] destination.write(content) - store_str("statchg", basename, content) + doc.store_str(basename, content) try: ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10 diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py index 052e89132f..693aafb385 100644 --- a/ietf/group/tasks.py +++ b/ietf/group/tasks.py @@ -44,10 +44,10 @@ def generate_wg_charters_files_task(): encoding="utf8", ) - with charters_file.open() as f: - store_file("indexes", "1wg-charters.txt", f) - with charters_by_acronym_file.open() as f: - store_file("indexes", "1wg-charters-by-acronym.txt", f) + with charters_file.open("rb") as f: + store_file("indexes", "1wg-charters.txt", f, allow_overwrite=True) + with charters_by_acronym_file.open("rb") as f: + store_file("indexes", "1wg-charters-by-acronym.txt", f, allow_overwrite=True) charter_copy_dests = [ getattr(settings, "CHARTER_COPY_PATH", None), @@ -109,7 +109,7 @@ def generate_wg_summary_files_task(): encoding="utf8", ) - with summary_file.open() as f: - store_file("indexes", "1wg-summary.txt", f) - with summary_by_acronym_file.open() as f: - store_file("indexes", "1wg-summary-by-acronym.txt", f) + with summary_file.open("rb") as f: + store_file("indexes", "1wg-summary.txt", f, allow_overwrite=True) + with summary_by_acronym_file.open("rb") as f: + store_file("indexes", "1wg-summary-by-acronym.txt", f, allow_overwrite=True) diff --git a/ietf/idindex/tasks.py b/ietf/idindex/tasks.py index e1ea83910b..2f5f1871d7 100644 --- a/ietf/idindex/tasks.py +++ b/ietf/idindex/tasks.py @@ -41,7 +41,7 @@ def move_into_place(self, src_path: Path, dest_path: Path, hardlink_dirs: List[P target.unlink(missing_ok=True) os.link(dest_path, target) # until python>=3.10 with dest_path.open("rb") as f: - store_file("indexes", dest_path.name, f) + store_file("indexes", dest_path.name, f, allow_overwrite=True) def cleanup(self): for tf_path in self.cleanup_list: diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index e2da5abd72..1af29044b3 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -21,7 +21,6 @@ import debug # pyflakes:ignore -from ietf.doc.storage_utils import store_file from ietf.ietfauth.utils import has_role from ietf.name.models import DocRelationshipName from ietf.liaisons.utils import get_person_for_user,is_authorized_individual @@ -381,7 +380,7 @@ def save_attachments(self): attach_file.write(attached_file.read()) attach_file.close() attached_file.seek(0) - store_file(attach.type_id, attach.uploaded_filename, attached_file) + attach.store_file(attach.uploaded_filename, attached_file) if not self.is_new: # create modified event diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index e4cd370e2e..e1d1e90b8d 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -20,7 +20,6 @@ import debug # pyflakes:ignore from ietf.doc.models import Document, State, NewRevisionDocEvent -from ietf.doc.storage_utils import store_str from ietf.group.models import Group from ietf.group.utils import groups_managed_by from ietf.meeting.models import Session, Meeting, Schedule, countries, timezones, TimeSlot, Room @@ -362,7 +361,7 @@ def save_agenda(self): os.makedirs(directory) with io.open(path, "w", encoding='utf-8') as file: file.write(self.cleaned_data['agenda']) - store_str("agenda", doc.uploaded_filename, self.cleaned_data['agenda']) + doc.store_str(doc.uploaded_filename, self.cleaned_data['agenda']) class InterimAnnounceForm(forms.ModelForm): diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 691ce923d4..465e694eb1 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -55,6 +55,7 @@ from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName from ietf.utils.decorators import skip_coverage from ietf.utils.mail import outbox, empty_outbox, get_payload_text +from ietf.utils.test_runner import TestBlobstoreManager from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.timezone import date_today, time_now @@ -5227,6 +5228,7 @@ def test_interim_request_options(self): def do_interim_request_single_virtual(self, emails_expected): make_meeting_test_data() + TestBlobstoreManager().emptyTestBlobstores() group = Group.objects.get(acronym='mars') date = date_today() + datetime.timedelta(days=30) time = time_now().replace(microsecond=0,second=0) @@ -5304,6 +5306,7 @@ def test_interim_request_single_virtual_settings_approval_not_required(self): def test_interim_request_single_in_person(self): make_meeting_test_data() + TestBlobstoreManager().emptyTestBlobstores() group = Group.objects.get(acronym='mars') date = date_today() + datetime.timedelta(days=30) time = time_now().replace(microsecond=0,second=0) @@ -5488,6 +5491,7 @@ def test_interim_request_multi_day_cancel(self): def test_interim_request_series(self): make_meeting_test_data() + TestBlobstoreManager().emptyTestBlobstores() meeting_count_before = Meeting.objects.filter(type='interim').count() date = date_today() + datetime.timedelta(days=30) if (date.month, date.day) == (12, 31): @@ -6118,6 +6122,7 @@ def strfdelta(self, tdelta, fmt): def test_interim_request_edit_agenda_updates_doc(self): """Updating the agenda through the request edit form should update the doc correctly""" make_interim_test_data() + TestBlobstoreManager().emptyTestBlobstores() meeting = add_event_info_to_session_qs(Session.objects.filter(meeting__type='interim', group__acronym='mars')).filter(current_status='sched').first().meeting group = meeting.session_set.first().group url = urlreverse('ietf.meeting.views.interim_request_edit', kwargs={'number': meeting.number}) @@ -6494,11 +6499,9 @@ def test_upload_minutes_agenda(self): text = doc.text() self.assertIn('Some text', text) self.assertNotIn('
', text) - self.assertIn('charset="utf-8"', text) text = retrieve_str(doctype, f"{doc.name}-{doc.rev}.html") self.assertIn('Some text', text) self.assertNotIn('
', text) - self.assertIn('charset="utf-8"', text) # txt upload test_bytes = b'This is some text for a test, with the word\nvirtual at the beginning of a line.' @@ -6972,6 +6975,7 @@ def test_approve_proposed_slides(self, mock_slides_manager_cls): @override_settings(MEETECHO_API_CONFIG="fake settings") # enough to trigger API calls @patch("ietf.meeting.views.SlidesManager") def test_approve_proposed_slides_multisession_apply_one(self, mock_slides_manager_cls): + TestBlobstoreManager().emptyTestBlobstores() submission = SlideSubmissionFactory(session__meeting__type_id='ietf') session1 = submission.session session2 = SessionFactory(group=submission.session.group, meeting=submission.session.meeting) @@ -7000,6 +7004,7 @@ def test_approve_proposed_slides_multisession_apply_one(self, mock_slides_manage @override_settings(MEETECHO_API_CONFIG="fake settings") # enough to trigger API calls @patch("ietf.meeting.views.SlidesManager") def test_approve_proposed_slides_multisession_apply_all(self, mock_slides_manager_cls): + TestBlobstoreManager().emptyTestBlobstores() submission = SlideSubmissionFactory(session__meeting__type_id='ietf') session1 = submission.session session2 = SessionFactory(group=submission.session.group, meeting=submission.session.meeting) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index dc9fb6457c..7f670f091e 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -778,6 +778,8 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N clean_bytes = clean.encode('utf8') destination.write(clean_bytes) # Assumes contents of subdir are always document type ids + # TODO-BLOBSTORE: see if we can refactor this so that the connection to the document isn't lost + # In the meantime, consider faking it by parsing filename (shudder). store_bytes(subdir, filename.name, clean_bytes) if request and clean != text: messages.warning(request, @@ -792,6 +794,7 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N file.seek(0) if hasattr(file, "chunks"): chunks = file.chunks() + # TODO-BLOBSTORE: See above question about refactoring store_bytes(subdir, filename.name, b"".join(chunks)) return None @@ -819,6 +822,7 @@ def new_doc_for_session(type_id, session): session.presentations.create(document=doc,rev='00') return doc +# TODO-BLOBSTORE - consider adding doc to this signature and factoring away type_id def write_doc_for_session(session, type_id, filename, contents): filename = Path(filename) path = Path(session.meeting.get_materials_path()) / type_id diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index dbb90a3e4c..8439bac866 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -52,7 +52,7 @@ from ietf.doc.fields import SearchableDocumentsField from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent -from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_bytes, store_file +from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_file from ietf.group.models import Group from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group from ietf.person.models import Person, User @@ -5096,7 +5096,7 @@ def approve_proposed_slides(request, slidesubmission_id, num): if not os.path.exists(path): os.makedirs(path) shutil.move(submission.staged_filepath(), os.path.join(path, target_filename)) - store_bytes("slides", target_filename, retrieve_bytes("staging", submission.filename)) + doc.store_bytes(target_filename, retrieve_bytes("staging", submission.filename)) remove_from_storage("staging", submission.filename) post_process(doc) DocEvent.objects.create(type="approved_slides", doc=doc, rev=doc.rev, by=request.user.person, desc="Slides approved") diff --git a/ietf/settings_test.py b/ietf/settings_test.py index a71dffb46a..8dfab1976a 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -108,7 +108,7 @@ def tempdir_with_cleanup(**kwargs): for storagename in MORE_STORAGE_NAMES: STORAGES[storagename] = { - "BACKEND": "storages.backends.s3.S3Storage", + "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", "OPTIONS": dict( endpoint_url="http://blobstore:9000", access_key="minio_root", diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index faaa038b32..9a993480cd 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -31,7 +31,7 @@ ReviewFactory, WgRfcFactory) from ietf.doc.models import ( Document, DocEvent, State, BallotPositionDocEvent, DocumentAuthor, SubmissionDocEvent ) -from ietf.doc.storage_utils import exists_in_storage, retrieve_str, store_file, store_str +from ietf.doc.storage_utils import exists_in_storage, retrieve_str, store_str from ietf.doc.utils import create_ballot_if_not_open, can_edit_docextresources, update_action_holders from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group @@ -54,6 +54,7 @@ from ietf.utils import tool_version from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, get_payload_text +from ietf.utils.test_runner import TestBlobstoreManager from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.timezone import date_today from ietf.utils.draft import PlaintextDraft @@ -356,6 +357,7 @@ def verify_bibxml_ids_creation(self, draft): def submit_new_wg(self, formats): # submit new -> supply submitter info -> approve + TestBlobstoreManager().emptyTestBlobstores() GroupFactory(type_id='wg',acronym='ames') mars = GroupFactory(type_id='wg', acronym='mars') RoleFactory(name_id='chair', group=mars, person__user__username='marschairman') @@ -543,6 +545,7 @@ def test_submit_new_wg_as_author_bad_submitter(self): def submit_new_concluded_wg_as_author(self, group_state_id='conclude'): """A new concluded WG submission by a logged-in author needs AD approval""" + TestBlobstoreManager().emptyTestBlobstores() mars = GroupFactory(type_id='wg', acronym='mars', state_id=group_state_id) draft = WgDraftFactory(group=mars) setup_default_community_list_for_group(draft.group) @@ -588,6 +591,7 @@ def test_submit_new_wg_with_extresources(self): def submit_existing(self, formats, change_authors=True, group_type='wg', stream_type='ietf'): # submit new revision of existing -> supply submitter info -> prev authors confirm + TestBlobstoreManager().emptyTestBlobstores() def _assert_authors_are_action_holders(draft, expect=True): for author in draft.authors(): @@ -924,6 +928,7 @@ def test_submit_existing_iab_with_extresources(self): def submit_new_individual(self, formats): # submit new -> supply submitter info -> confirm + TestBlobstoreManager().emptyTestBlobstores() name = "draft-authorname-testing-tests" rev = "00" @@ -1009,6 +1014,7 @@ def test_submit_new_individual_txt_xml(self): self.submit_new_individual(["txt", "xml"]) def submit_new_draft_no_org_or_address(self, formats): + TestBlobstoreManager().emptyTestBlobstores() name = 'draft-testing-no-org-or-address' author = PersonFactory() @@ -1099,6 +1105,7 @@ def _assert_extresource_change_event(self, doc, is_present=True): self.assertIsNone(event, 'External resource change event was unexpectedly created') def submit_new_draft_with_extresources(self, group): + TestBlobstoreManager().emptyTestBlobstores() name = 'draft-testing-with-extresources' status_url, author = self.do_submission(name, rev='00', group=group) @@ -1128,6 +1135,7 @@ def test_submit_new_individual_with_extresources(self): def submit_new_individual_logged_in(self, formats): # submit new -> supply submitter info -> done + TestBlobstoreManager().emptyTestBlobstores() name = "draft-authorname-testing-logged-in" rev = "00" @@ -1271,6 +1279,7 @@ def submit_existing_with_extresources(self, group_type, stream_type='ietf'): Unlike some other tests in this module, does not confirm draft if this would be required. """ + TestBlobstoreManager().emptyTestBlobstores() orig_draft: Document = DocumentFactory( # type: ignore[annotation-unchecked] type_id='draft', group=GroupFactory(type_id=group_type) if group_type else None, @@ -1311,6 +1320,7 @@ def test_submit_update_individual_with_extresources(self): def submit_new_individual_replacing_wg(self, logged_in=False, group_state_id='active', notify_ad=False): """Chair of an active WG should be notified if individual draft is proposed to replace a WG draft""" + TestBlobstoreManager().emptyTestBlobstores() name = "draft-authorname-testing-tests" rev = "00" group = None @@ -1928,6 +1938,7 @@ def do_wg_approval_auth_test(self, state, chair_can_approve=False): Assumes approval allowed by AD and secretary and, optionally, chair of WG """ + TestBlobstoreManager().emptyTestBlobstores() class _SubmissionFactory: """Helper class to generate fresh submissions""" def __init__(self, author, state): @@ -2777,6 +2788,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase): """Tests of async submission-related tasks""" def test_process_and_accept_uploaded_submission(self): """process_and_accept_uploaded_submission should properly process a submission""" + TestBlobstoreManager().emptyTestBlobstores() _today = date_today() xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml') xml_data = xml.read() @@ -2792,7 +2804,7 @@ def test_process_and_accept_uploaded_submission(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(xml_data) - store_str("staging", "draft-somebpdy-test-00.xml", xml_data) + store_str("staging", "draft-somebody-test-00.xml", xml_data) txt_path = xml_path.with_suffix('.txt') self.assertFalse(txt_path.exists()) html_path = xml_path.with_suffix('.html') @@ -2830,6 +2842,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): txt.close() # submitter is not an author + TestBlobstoreManager().emptyTestBlobstores() submitter = PersonFactory() submission = SubmissionFactory( name='draft-somebody-test', @@ -2848,6 +2861,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): self.assertIn('not one of the document authors', submission.submissionevent_set.last().desc) # author has no email address in XML + TestBlobstoreManager().emptyTestBlobstores() submission = SubmissionFactory( name='draft-somebody-test', rev='00', @@ -2865,6 +2879,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): self.assertIn('Email address not found for all authors', submission.submissionevent_set.last().desc) # no title + TestBlobstoreManager().emptyTestBlobstores() submission = SubmissionFactory( name='draft-somebody-test', rev='00', @@ -2882,6 +2897,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): self.assertIn('Could not extract a valid title', submission.submissionevent_set.last().desc) # draft name mismatch + TestBlobstoreManager().emptyTestBlobstores() submission = SubmissionFactory( name='draft-different-name', rev='00', @@ -2899,6 +2915,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): self.assertIn('Submission rejected: XML Internet-Draft filename', submission.submissionevent_set.last().desc) # rev mismatch + TestBlobstoreManager().emptyTestBlobstores() submission = SubmissionFactory( name='draft-somebody-test', rev='01', @@ -2916,6 +2933,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): self.assertIn('Submission rejected: XML Internet-Draft revision', submission.submissionevent_set.last().desc) # not xml + TestBlobstoreManager().emptyTestBlobstores() submission = SubmissionFactory( name='draft-somebody-test', rev='00', @@ -2933,6 +2951,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): self.assertIn('Only XML Internet-Draft submissions', submission.submissionevent_set.last().desc) # wrong state + TestBlobstoreManager().emptyTestBlobstores() submission = SubmissionFactory( name='draft-somebody-test', rev='00', @@ -2951,6 +2970,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): self.assertEqual(submission.state_id, 'uploaded', 'State should not be changed') # failed checker + TestBlobstoreManager().emptyTestBlobstores() submission = SubmissionFactory( name='draft-somebody-test', rev='00', @@ -2998,6 +3018,7 @@ def test_process_and_accept_uploaded_submission_task_ignores_invalid_id(self, mo self.assertEqual(mock_method.call_count, 0) def test_process_submission_xml(self): + TestBlobstoreManager().emptyTestBlobstores() xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.xml" xml, _ = submission_file( "draft-somebody-test-00", @@ -3024,27 +3045,32 @@ def test_process_submission_xml(self): self.assertEqual(output["xml_version"], "3") # Should behave on missing or partial elements + TestBlobstoreManager().emptyTestBlobstores() xml_path.write_text(re.sub(r"", "", xml_contents)) # strip entirely store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", "", xml_contents)) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], None) + TestBlobstoreManager().emptyTestBlobstores() xml_path.write_text(re.sub(r")", r"\1 day=\2", xml_contents)) # remove month store_str("staging", "draft-somebody-test-00.xml", re.sub(r"()", r"\1 day=\2", xml_contents)) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], date_today()) + TestBlobstoreManager().emptyTestBlobstores() xml_path.write_text(re.sub(r"", r"", xml_contents)) # remove day store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", r"", xml_contents)) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], date_today()) # name mismatch + TestBlobstoreManager().emptyTestBlobstores() xml, _ = submission_file( "draft-somebody-wrong-name-00", # name that appears in the file "draft-somebody-test-00.xml", @@ -3054,11 +3080,12 @@ def test_process_submission_xml(self): ) xml_path.write_text(xml.read()) xml.seek(0) - store_file("staging", "draft-somebody-test-00.xml", xml) + store_str("staging", "draft-somebody-test-00.xml", xml.read()) with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"): process_submission_xml("draft-somebody-test", "00") # rev mismatch + TestBlobstoreManager().emptyTestBlobstores() xml, _ = submission_file( "draft-somebody-test-01", # name that appears in the file "draft-somebody-test-00.xml", @@ -3068,11 +3095,12 @@ def test_process_submission_xml(self): ) xml_path.write_text(xml.read()) xml.seek(0) - store_file("staging", "draft-somebody-test-00.xml", xml) + store_str("staging", "draft-somebody-test-00.xml", xml.read()) with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"): process_submission_xml("draft-somebody-test", "00") # missing title + TestBlobstoreManager().emptyTestBlobstores() xml, _ = submission_file( "draft-somebody-test-00", # name that appears in the file "draft-somebody-test-00.xml", @@ -3082,11 +3110,12 @@ def test_process_submission_xml(self): ) xml_path.write_text(xml.read()) xml.seek(0) - store_file("staging", "draft-somebody-test-00.xml", xml) + store_str("staging", "draft-somebody-test-00.xml", xml.read()) with self.assertRaisesMessage(SubmissionError, "Could not extract a valid title"): process_submission_xml("draft-somebody-test", "00") def test_process_submission_text(self): + TestBlobstoreManager().emptyTestBlobstores() txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.txt" txt, _ = submission_file( "draft-somebody-test-00", @@ -3097,7 +3126,7 @@ def test_process_submission_text(self): ) txt_path.write_text(txt.read()) txt.seek(0) - store_file("staging", "draft-somebody-test-00.txt", txt) + store_str("staging", "draft-somebody-test-00.txt", txt.read()) output = process_submission_text("draft-somebody-test", "00") self.assertEqual(output["filename"], "draft-somebody-test") self.assertEqual(output["rev"], "00") @@ -3113,6 +3142,7 @@ def test_process_submission_text(self): self.assertIsNone(output["xml_version"]) # name mismatch + TestBlobstoreManager().emptyTestBlobstores() txt, _ = submission_file( "draft-somebody-wrong-name-00", # name that appears in the file "draft-somebody-test-00.txt", @@ -3123,12 +3153,13 @@ def test_process_submission_text(self): with txt_path.open('w') as fd: fd.write(txt.read()) txt.seek(0) - store_file("staging", "draft-somebody-test-00.txt", txt) + store_str("staging", "draft-somebody-test-00.txt", txt.read()) txt.close() with self.assertRaisesMessage(SubmissionError, 'disagrees with submission filename'): process_submission_text("draft-somebody-test", "00") # rev mismatch + TestBlobstoreManager().emptyTestBlobstores() txt, _ = submission_file( "draft-somebody-test-01", # name that appears in the file "draft-somebody-test-00.txt", @@ -3139,7 +3170,7 @@ def test_process_submission_text(self): with txt_path.open('w') as fd: fd.write(txt.read()) txt.seek(0) - store_file("staging", "draft-somebody-test-00.txt", txt) + store_str("staging", "draft-somebody-test-00.txt", txt.read()) txt.close() with self.assertRaisesMessage(SubmissionError, 'disagrees with submission revision'): process_submission_text("draft-somebody-test", "00") diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 69bef0db75..b240c337fa 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -456,6 +456,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): from ietf.doc.expire import move_draft_files_to_archive move_draft_files_to_archive(draft, prev_rev) + submission.draft = draft move_files_to_repository(submission) submission.state = DraftSubmissionStateName.objects.get(slug="posted") log.log(f"{submission.name}: moved files") @@ -489,7 +490,6 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): if new_possibly_replaces: send_review_possibly_replaces_request(request, draft, submitter_info) - submission.draft = draft submission.save() create_submission_event(request, submission, approved_subm_desc) @@ -671,7 +671,7 @@ def move_files_to_repository(submission): # authoritative, the source and dest checks will need to apply to the stores instead. content_bytes = retrieve_bytes("staging", fname) store_bytes("active-draft", f"{ext}/{fname}", content_bytes) - store_bytes("draft", f"{ext}/{fname}", content_bytes) + submission.draft.store_bytes(f"{ext}/{fname}", content_bytes) remove_from_storage("staging", fname) elif dest.exists(): log.log("Intended to move '%s' to '%s', but found source missing while destination exists.") @@ -689,7 +689,7 @@ def remove_staging_files(name, rev, exts=None): basename = pathlib.Path(settings.IDSUBMIT_STAGING_PATH) / f'{name}-{rev}' for ext in exts: basename.with_suffix(ext).unlink(missing_ok=True) - remove_from_storage("staging", basename.with_suffix(ext).name) + remove_from_storage("staging", basename.with_suffix(ext).name, warn_if_missing=False) def remove_submission_files(submission): @@ -1002,8 +1002,10 @@ def render_missing_formats(submission): xml_version, ) ) - with Path(txt_path).open() as f: - store_file("staging", f"{submission.name}-{submission.rev}.txt", f) + # When the blobstores become autoritative - the guard at the + # containing if statement needs to be based on the store + with Path(txt_path).open("rb") as f: + store_file("staging", f"{submission.name}-{submission.rev}.txt", f) # --- Convert to html --- html_path = staging_path(submission.name, submission.rev, '.html') @@ -1026,7 +1028,7 @@ def render_missing_formats(submission): xml_version, ) ) - with Path(html_path).open() as f: + with Path(html_path).open("rb") as f: store_file("staging", f"{submission.name}-{submission.rev}.html", f) @@ -1379,6 +1381,7 @@ def process_and_validate_submission(submission): except SubmissionError: raise # pass SubmissionErrors up the stack except Exception as err: + # (this is a good point to just `raise err` when diagnosing Submission test failures) # convert other exceptions into SubmissionErrors log.log(f'Unexpected exception while processing submission {submission.pk}.') log.log(traceback.format_exc()) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 3412fc633c..437d201258 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -753,7 +753,8 @@ def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_versio # contains parent classes to later subclasses, the parent classes will determine the ordering, so use the most # specific classes necessary to get the right ordering: self.reorder_by = (PyFlakesTestCase, MyPyTest,) + self.reorder_by + (StaticLiveServerTestCase, TemplateTagTest, CoverageTest,) - self.buckets = set() + #self.buckets = set() + self.blobstoremanager = TestBlobstoreManager() def setup_test_environment(self, **kwargs): global template_coverage_collection @@ -938,27 +939,8 @@ def setup_test_environment(self, **kwargs): print(" (extra pedantically)") self.vnu = start_vnu_server() - blobstore = boto3.resource("s3", - endpoint_url="http://blobstore:9000", - aws_access_key_id="minio_root", - aws_secret_access_key="minio_pass", - aws_session_token=None, - config=boto3.session.Config(signature_version="s3v4"), - #config=boto3.session.Config(signature_version=botocore.UNSIGNED), - verify=False - ) - for storagename in settings.MORE_STORAGE_NAMES: - bucketname = f"test-{storagename}" - try: - bucket = blobstore.create_bucket(Bucket=bucketname) - #debug.show('f"created {bucket}"') - self.buckets.add(bucket) - except blobstore.meta.client.exceptions.BucketAlreadyOwnedByYou: - #debug.show('f"{bucketname} already there"') - bucket = blobstore.Bucket(bucketname) - self.buckets.add(bucket) - - + self.blobstoremanager.createTestBlobstores() + super(IetfTestRunner, self).setup_test_environment(**kwargs) def teardown_test_environment(self, **kwargs): @@ -989,16 +971,7 @@ def teardown_test_environment(self, **kwargs): if self.vnu: self.vnu.terminate() - - for bucket in self.buckets: - # bucketname=bucket.name - # debug.show('f"Trying to delete {bucketname} contents"') - # debug.show("bucket.objects.delete()") - # debug.show('f"Trying to delete {bucketname} itself"') - # debug.show("bucket.delete()") - bucket.objects.delete() - bucket.delete() - + self.blobstoremanager.destroyTestBlobstores() super(IetfTestRunner, self).teardown_test_environment(**kwargs) @@ -1254,3 +1227,39 @@ def tearDown(self): for k, v in self.replaced_settings.items(): setattr(settings, k, v) super().tearDown() + +class TestBlobstoreManager(): + # N.B. buckets and blobstore are intentional Class-level attributes + buckets = set() + + blobstore = boto3.resource("s3", + endpoint_url="http://blobstore:9000", + aws_access_key_id="minio_root", + aws_secret_access_key="minio_pass", + aws_session_token=None, + config=boto3.session.Config(signature_version="s3v4"), + #config=boto3.session.Config(signature_version=botocore.UNSIGNED), + verify=False + ) + + def createTestBlobstores(self): + for storagename in settings.MORE_STORAGE_NAMES: + bucketname = f"test-{storagename}" + try: + bucket = self.blobstore.create_bucket(Bucket=bucketname) + self.buckets.add(bucket) + except self.blobstore.meta.client.exceptions.BucketAlreadyOwnedByYou: + bucket = self.blobstore.Bucket(bucketname) + self.buckets.add(bucket) + + def destroyTestBlobstores(self): + self.emptyTestBlobstores(destroy=True) + + def emptyTestBlobstores(self, destroy=False): + # debug.show('f"Asked to empty test blobstores with destroy={destroy}"') + for bucket in self.buckets: + bucket.objects.delete() + if destroy: + bucket.delete() + if destroy: + self.buckets = set() From f5d59bc53ea7a70e72567827ac75a68f904d58e2 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 6 Feb 2025 12:11:26 -0600 Subject: [PATCH 54/87] fix: deburr some debugging code --- ietf/doc/storage_backends.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 07cd14ec80..5951288608 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -35,8 +35,7 @@ def store_file( if not allow_overwrite and not is_new: log(f"Failed to save {kind}:{name} - name already exists in store") debug.show('f"Failed to save {kind}:{name} - name already exists in store"') - debug.traceback() - raise Exception("Not ignoring overwrite attempts while testing") + # raise Exception("Not ignoring overwrite attempts while testing") else: try: new_name = self.save(name, file) @@ -69,11 +68,9 @@ def store_file( except Exception as e: # Log and then swallow the exception while we're learning. # Don't let failure pass so quietly when these are the autoritative bits. - log(f"Failed to save {kind}:{name}", e) - raise e - debug.show("type(e)") - debug.show("e") - debug.traceback() + complaint = f"Failed to save {kind}:{name}" + log(complaint, e) + debug.show('f"{complaint}: {e}') finally: del self.in_flight_custom_metadata[name] return None From 1f90170dd02b6792b795eec783343d83e6d1d816 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 11 Feb 2025 15:43:14 -0600 Subject: [PATCH 55/87] fix: only set the deleted timestamp once --- ietf/doc/storage_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 5951288608..63000821dd 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -107,7 +107,7 @@ def remove_from_storage( debug.show("complaint") else: # Note that existing_record is a queryset that will have one matching object - existing_record.update(deleted=now) + existing_record.filter(deleted__isnull=True).update(deleted=now) def _get_write_parameters(self, name, content=None): # debug.show('f"getting write parameters for {name}"') From 9194b22e818472ae9a35638b27657fb38bbbadad Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 11 Feb 2025 16:07:36 -0600 Subject: [PATCH 56/87] chore: correct typo --- ietf/doc/storage_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 63000821dd..d91bfb8a3a 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -96,7 +96,7 @@ def remove_from_storage( except FileNotFoundError: if warn_if_missing: complaint = ( - f"WARNING: Asked to delete non-existant {name} from {kind} storage" + f"WARNING: Asked to delete non-existent {name} from {kind} storage" ) log(complaint) debug.show("complaint") From 9cf8c02670930794e9dac1fb8b64ac29e31de876 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 11 Feb 2025 16:26:28 -0600 Subject: [PATCH 57/87] fix: get_or_create vs get and test --- ietf/doc/storage_backends.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index d91bfb8a3a..7564973e1b 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -40,26 +40,24 @@ def store_file( try: new_name = self.save(name, file) now = timezone.now() - existing_record = StoredObject.objects.filter(store=kind, name=name) - if existing_record.exists(): - # Note this is updating a queryset which is guaranteed by constraints to have one object - existing_record.update( - sha384=self.in_flight_custom_metadata[name]["sha384"], - len=int(self.in_flight_custom_metadata[name]["len"]), - modified=now, - ) - else: - StoredObject.objects.create( - store=kind, - name=name, + record, created = StoredObject.objects.get_or_create( + store=kind, + name=name, + defaults=dict( sha384=self.in_flight_custom_metadata[name]["sha384"], len=int(self.in_flight_custom_metadata[name]["len"]), store_created=now, created=now, modified=now, - doc_name=doc_name, - doc_rev=doc_rev, + doc_name=doc_name, # Note that these are assumed to be invariant + doc_rev=doc_rev, # for a given name ) + ) + if not created: + record.sha384=self.in_flight_custom_metadata[name]["sha384"] + record.len=int(self.in_flight_custom_metadata[name]["len"]) + record.modified=now + record.save() if new_name != name: complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." log(complaint) @@ -70,7 +68,7 @@ def store_file( # Don't let failure pass so quietly when these are the autoritative bits. complaint = f"Failed to save {kind}:{name}" log(complaint, e) - debug.show('f"{complaint}: {e}') + debug.show('f"{complaint}: {e}"') finally: del self.in_flight_custom_metadata[name] return None From 0e40c525c248d4b43c656abad1c010fd03d3f46e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 11 Feb 2025 17:01:52 -0600 Subject: [PATCH 58/87] fix: avoid the questionable is_seekable helper --- ietf/doc/storage_backends.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 7564973e1b..8056c32918 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -5,7 +5,6 @@ from hashlib import sha384 from io import BufferedReader from storages.backends.s3 import S3Storage -from storages.utils import is_seekable from typing import Dict, Optional, Union from django.core.files.base import File @@ -112,11 +111,11 @@ def _get_write_parameters(self, name, content=None): params = super()._get_write_parameters(name, content) if "Metadata" not in params: params["Metadata"] = {} - if not is_seekable(content): - # TODO-BLOBSTORE + try: + content.seek(0) + except AttributeError: # TODO-BLOBSTORE debug.say("Encountered Non-Seekable content") raise NotImplementedError("cannot handle unseekable content") - content.seek(0) content_bytes = content.read() if not isinstance( content_bytes, bytes From 3721007e377d08519963e52884c762f8c12c90d1 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 11 Feb 2025 17:14:48 -0600 Subject: [PATCH 59/87] chore: capture future design consideration --- ietf/doc/storage_backends.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 8056c32918..bab07cacd7 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -14,6 +14,10 @@ from ietf.utils.timezone import timezone + +# TODO-BLOBSTORE +# Consider overriding save directly so that +# we capture metadata for, e.g., ImageField objects class CustomS3Storage(S3Storage): def __init__(self, **settings): From 1578e3bfc39f4df850e19dd19f9c8d799b756570 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 12 Feb 2025 12:31:39 -0400 Subject: [PATCH 60/87] chore: blob store cfg for k8s --- k8s/settings_local.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index f266ffcd62..20de7d04b9 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -285,3 +285,32 @@ def _multiline_to_list(s): # Console logs as JSON instead of plain when running in k8s LOGGING["handlers"]["console"]["formatter"] = "json" + +# Configure storages for the blob store +_blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL") +_blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY") +_blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY") +if None in (_blob_store_endpoint_url, _blob_store_access_key, _blob_store_secret_key): + raise RuntimeError( + "All of DATATRACKER_BLOB_STORE_ENDPOINT_URL, DATATRACKER_BLOB_STORE_ACCESS_KEY, " + "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set" + ) +try: + from ietf.settings import MORE_STORAGE_NAMES +except ImportError: + pass # Don't fail if MORE_STORAGE_NAMES is not there, just don't configure it +else: + from ietf.settings import boto3, STORAGES # do fail if these aren't found! + for storage_name in MORE_STORAGE_NAMES: + STORAGES[storage_name] = { + "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", + "OPTIONS": dict( + endpoint_url=_blob_store_endpoint_url, + access_key=_blob_store_access_key, + secret_key=_blob_store_secret_key, + security_token=None, + client_config=boto3.session.Config(signature_version="s3v4"), + verify=False, + bucket_name=storage_name, + ), + } From abd1e76941a3db9a0bc295038a0e08f64e6600ea Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 13 Feb 2025 11:08:41 -0600 Subject: [PATCH 61/87] chore: black --- ietf/doc/storage_backends.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index bab07cacd7..fa43e0ebac 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -14,7 +14,6 @@ from ietf.utils.timezone import timezone - # TODO-BLOBSTORE # Consider overriding save directly so that # we capture metadata for, e.g., ImageField objects @@ -44,7 +43,7 @@ def store_file( new_name = self.save(name, file) now = timezone.now() record, created = StoredObject.objects.get_or_create( - store=kind, + store=kind, name=name, defaults=dict( sha384=self.in_flight_custom_metadata[name]["sha384"], @@ -52,14 +51,14 @@ def store_file( store_created=now, created=now, modified=now, - doc_name=doc_name, # Note that these are assumed to be invariant - doc_rev=doc_rev, # for a given name - ) + doc_name=doc_name, # Note that these are assumed to be invariant + doc_rev=doc_rev, # for a given name + ), ) if not created: - record.sha384=self.in_flight_custom_metadata[name]["sha384"] - record.len=int(self.in_flight_custom_metadata[name]["len"]) - record.modified=now + record.sha384 = self.in_flight_custom_metadata[name]["sha384"] + record.len = int(self.in_flight_custom_metadata[name]["len"]) + record.modified = now record.save() if new_name != name: complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." @@ -117,7 +116,7 @@ def _get_write_parameters(self, name, content=None): params["Metadata"] = {} try: content.seek(0) - except AttributeError: # TODO-BLOBSTORE + except AttributeError: # TODO-BLOBSTORE debug.say("Encountered Non-Seekable content") raise NotImplementedError("cannot handle unseekable content") content_bytes = content.read() From a1c732f507a1ec1f6866211e2b7d3eb9f7aaa90d Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 13 Feb 2025 11:10:46 -0600 Subject: [PATCH 62/87] chore: copyright --- .../0025_storedobject_storedobject_unique_name_per_store.py | 2 +- .../0010_alter_floorplan_image_alter_meetinghost_logo.py | 2 +- .../0004_alter_person_photo_alter_person_photo_thumb.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py b/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py index 9161007d4e..e948ca3011 100644 --- a/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py +++ b/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.18 on 2025-02-04 20:51 +# Copyright The IETF Trust 2025, All Rights Reserved from django.db import migrations, models diff --git a/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py index 7d9a92b12d..594a1a4048 100644 --- a/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py +++ b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.18 on 2025-01-29 14:49 +# Copyright The IETF Trust 2025, All Rights Reserved from django.db import migrations, models import ietf.meeting.models diff --git a/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py index 63c535af07..f34382fa70 100644 --- a/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py +++ b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.18 on 2025-01-29 14:49 +# Copyright The IETF Trust 2025, All Rights Reserved from django.db import migrations, models import ietf.utils.storage From d993fd57044ff66cf106a381a9e0a35f61795532 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Feb 2025 16:44:14 -0400 Subject: [PATCH 63/87] ci: bucket name prefix option + run Black Adds/uses DATATRACKER_BLOB_STORE_BUCKET_PREFIX option. Other changes are just Black styling. --- k8s/settings_local.py | 64 +++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 20de7d04b9..344f94ffa5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -6,7 +6,7 @@ import json from ietf import __release_hash__ -from ietf.settings import * # pyflakes:ignore +from ietf.settings import * # pyflakes:ignore def _multiline_to_list(s): @@ -29,7 +29,7 @@ def _multiline_to_list(s): if _SECRET_KEY is not None: SECRET_KEY = _SECRET_KEY else: - raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") _NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) if _NOMCOM_APP_SECRET_B64 is not None: @@ -41,7 +41,7 @@ def _multiline_to_list(s): if _IANA_SYNC_PASSWORD is not None: IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD else: - raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") _RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) if _RFC_EDITOR_SYNC_PASSWORD is not None: @@ -59,25 +59,25 @@ def _multiline_to_list(s): if _GITHUB_BACKUP_API_KEY is not None: GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY else: - raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") _API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) if _API_KEY_TYPE is not None: API_KEY_TYPE = _API_KEY_TYPE else: - raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") _API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) if _API_PUBLIC_KEY_PEM_B64 is not None: API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) else: - raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") _API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) if _API_PRIVATE_KEY_PEM_B64 is not None: API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) else: - raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" @@ -102,7 +102,9 @@ def _multiline_to_list(s): # Configure persistent connections. A setting of 0 is Django's default. _conn_max_age = os.environ.get("DATATRACKER_DB_CONN_MAX_AGE", "0") # A string "none" means unlimited age. -DATABASES["default"]["CONN_MAX_AGE"] = None if _conn_max_age.lower() == "none" else int(_conn_max_age) +DATABASES["default"]["CONN_MAX_AGE"] = ( + None if _conn_max_age.lower() == "none" else int(_conn_max_age) +) # Enable connection health checks if DATATRACKER_DB_CONN_HEALTH_CHECK is the string "true" _conn_health_checks = bool( os.environ.get("DATATRACKER_DB_CONN_HEALTH_CHECKS", "false").lower() == "true" @@ -114,9 +116,11 @@ def _multiline_to_list(s): if _admins_str is not None: ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] else: - raise RuntimeError("DATATRACKER_ADMINS must be set") + raise RuntimeError("DATATRACKER_ADMINS must be set") -USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" +USING_DEBUG_EMAIL_SERVER = ( + os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" +) EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) @@ -126,7 +130,7 @@ def _multiline_to_list(s): CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), password=_celery_password, - queue=os.environ.get("RABBITMQ_QUEUE", "dt") + queue=os.environ.get("RABBITMQ_QUEUE", "dt"), ) IANA_SYNC_USERNAME = "ietfsync" @@ -140,10 +144,10 @@ def _multiline_to_list(s): raise RuntimeError("DATATRACKER_REGISTRATION_API_KEY must be set") STATS_REGISTRATION_ATTENDEES_JSON_URL = f"https://registration.ietf.org/{{number}}/attendees/?apikey={_registration_api_key}" -#FIRST_CUTOFF_DAYS = 12 -#SECOND_CUTOFF_DAYS = 12 -#SUBMISSION_CUTOFF_DAYS = 26 -#SUBMISSION_CORRECTION_DAYS = 57 +# FIRST_CUTOFF_DAYS = 12 +# SECOND_CUTOFF_DAYS = 12 +# SUBMISSION_CUTOFF_DAYS = 26 +# SUBMISSION_CORRECTION_DAYS = 57 MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 @@ -155,7 +159,7 @@ def _multiline_to_list(s): if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: MEETECHO_API_CONFIG = { "api_base": os.environ.get( - "DATATRACKER_MEETECHO_API_BASE", + "DATATRACKER_MEETECHO_API_BASE", "https://meetings.conf.meetecho.com/api/v1/", ), "client_id": _MEETECHO_CLIENT_ID, @@ -173,7 +177,9 @@ def _multiline_to_list(s): raise RuntimeError( "Only one of DATATRACKER_APP_API_TOKENS_JSON and DATATRACKER_APP_API_TOKENS_JSON_B64 may be set" ) - _APP_API_TOKENS_JSON = b64decode(os.environ.get("DATATRACKER_APP_API_TOKENS_JSON_B64")) + _APP_API_TOKENS_JSON = b64decode( + os.environ.get("DATATRACKER_APP_API_TOKENS_JSON_B64") + ) else: _APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) @@ -189,7 +195,9 @@ def _multiline_to_list(s): # Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting if "DATATRACKER_MATOMO_SITE_ID" in os.environ: - MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") + MATOMO_DOMAIN_PATH = os.environ.get( + "DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org" + ) MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") MATOMO_DISABLE_COOKIES = True @@ -197,9 +205,13 @@ def _multiline_to_list(s): _SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) if _SCOUT_KEY is not None: if SERVER_MODE == "production": - PROD_PRE_APPS = ["scout_apm.django", ] + PROD_PRE_APPS = [ + "scout_apm.django", + ] else: - DEV_PRE_APPS = ["scout_apm.django", ] + DEV_PRE_APPS = [ + "scout_apm.django", + ] SCOUT_MONITOR = True SCOUT_KEY = _SCOUT_KEY SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") @@ -216,16 +228,17 @@ def _multiline_to_list(s): STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) if STATIC_URL is None: from ietf import __version__ + STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" # Set these to the same as "production" in settings.py, whether production mode or not MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" -MEDIA_URL = "https://www.ietf.org/lib/dt/media/" +MEDIA_URL = "https://www.ietf.org/lib/dt/media/" PHOTOS_DIRNAME = "photo" PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME # Normally only set for debug, but needed until we have a real FS -DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, "static/dist-neue/manifest.json") # Binaries that are different in the docker image DE_GFM_BINARY = "/usr/local/bin/de-gfm" @@ -235,6 +248,7 @@ def _multiline_to_list(s): MEMCACHED_HOST = os.environ.get("DT_MEMCACHED_SERVICE_HOST", "127.0.0.1") MEMCACHED_PORT = os.environ.get("DT_MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ + CACHES = { "default": { "BACKEND": "ietf.utils.cache.LenientMemcacheCache", @@ -295,12 +309,16 @@ def _multiline_to_list(s): "All of DATATRACKER_BLOB_STORE_ENDPOINT_URL, DATATRACKER_BLOB_STORE_ACCESS_KEY, " "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set" ) +_blob_store_bucket_prefix = os.environ.get( + "DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "" +) try: from ietf.settings import MORE_STORAGE_NAMES except ImportError: pass # Don't fail if MORE_STORAGE_NAMES is not there, just don't configure it else: from ietf.settings import boto3, STORAGES # do fail if these aren't found! + for storage_name in MORE_STORAGE_NAMES: STORAGES[storage_name] = { "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", @@ -311,6 +329,6 @@ def _multiline_to_list(s): security_token=None, client_config=boto3.session.Config(signature_version="s3v4"), verify=False, - bucket_name=storage_name, + bucket_name="{_blob_store_bucket_prefix}{storage_name}".strip(), ), } From ac5f758025fee45be0e4539d253063034b645e2b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Feb 2025 17:56:15 -0400 Subject: [PATCH 64/87] ci: fix typo in bucket name expression --- k8s/settings_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 344f94ffa5..b239a5928e 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -329,6 +329,6 @@ def _multiline_to_list(s): security_token=None, client_config=boto3.session.Config(signature_version="s3v4"), verify=False, - bucket_name="{_blob_store_bucket_prefix}{storage_name}".strip(), + bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), ), } From a10cb6d03f518e3f5aa14ff6a91e4d619e7c3c92 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Feb 2025 17:56:59 -0400 Subject: [PATCH 65/87] chore: parameters in app-configure-blobstore Allows use with other blob stores. --- docker/scripts/app-configure-blobstore.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index b8908a8b3b..c8d5f40556 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -2,6 +2,7 @@ # Copyright The IETF Trust 2024, All Rights Reserved import boto3 +import os import sys from ietf.settings import MORE_STORAGE_NAMES @@ -10,15 +11,17 @@ def init_blobstore(): blobstore = boto3.resource( "s3", - endpoint_url="http://blobstore:9000", - aws_access_key_id="minio_root", - aws_secret_access_key="minio_pass", + endpoint_url=os.environ.get("BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000"), + aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"), + aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"), aws_session_token=None, config=boto3.session.Config(signature_version="s3v4"), verify=False, ) for bucketname in MORE_STORAGE_NAMES: - blobstore.create_bucket(Bucket=bucketname) + blobstore.create_bucket( + Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip() + ) if __name__ == "__main__": From 6f9b461ae9c36ed27d5d1006702ebddd94d4ab4e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Feb 2025 21:00:01 -0400 Subject: [PATCH 66/87] ci: remove verify=False option --- k8s/settings_local.py | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index b239a5928e..4b73c1b1e1 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -328,7 +328,6 @@ def _multiline_to_list(s): secret_key=_blob_store_secret_key, security_token=None, client_config=boto3.session.Config(signature_version="s3v4"), - verify=False, bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), ), } From 19f2995d69d7c3c5d0708473916725586ac3acdf Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Feb 2025 21:07:15 -0400 Subject: [PATCH 67/87] fix: don't return value from __init__ --- ietf/doc/storage_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index bab07cacd7..02c719c51b 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -22,7 +22,7 @@ class CustomS3Storage(S3Storage): def __init__(self, **settings): self.in_flight_custom_metadata: Dict[str, Dict[str, str]] = {} - return super().__init__(**settings) + super().__init__(**settings) def store_file( self, From 892117f09d98b8f12f187ca46161d7cb47454f3f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Feb 2025 21:43:09 -0400 Subject: [PATCH 68/87] feat: option to log timing of S3Storage calls --- ietf/doc/storage_backends.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 02c719c51b..96dbe0911d 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -24,6 +24,36 @@ def __init__(self, **settings): self.in_flight_custom_metadata: Dict[str, Dict[str, str]] = {} super().__init__(**settings) + def get_default_settings(self): + # add a default for the ietf_log_blob_timing boolean + return super().get_default_settings() | {"ietf_log_blob_timing": False} + + def _save(self, name, content): + # Only overriding this to add the option to time the operation + before = timezone.now() + result = super()._save(name, content) + if self.ietf_log_blob_timing: + dt = timezone.now() - before + log(f"S3Storage timing: _save('{name}', ...) for {self.bucket_name} took {dt.total_seconds()}") + return result + + def _open(self, name, mode="rb"): + # Only overriding this to add the option to time the operation + before = timezone.now() + result = super()._open(name, mode) + if self.ietf_log_blob_timing: + dt = timezone.now() - before + log(f"S3Storage timing: _open('{name}', ...) for {self.bucket_name} took {dt.total_seconds()}") + return result + + def delete(self, name): + # Only overriding this to add the option to time the operation + before = timezone.now() + super().delete(name) + if self.ietf_log_blob_timing: + dt = timezone.now() - before + log(f"S3Storage timing: delete('{name}') for {self.bucket_name} took {dt.total_seconds()}") + def store_file( self, kind: str, From 4be1bdffdc99609bcbdb8ffff4245334f9612c2d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Feb 2025 21:45:47 -0400 Subject: [PATCH 69/87] chore: units --- ietf/doc/storage_backends.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 96dbe0911d..63381a0201 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -34,7 +34,7 @@ def _save(self, name, content): result = super()._save(name, content) if self.ietf_log_blob_timing: dt = timezone.now() - before - log(f"S3Storage timing: _save('{name}', ...) for {self.bucket_name} took {dt.total_seconds()}") + log(f"S3Storage timing: _save('{name}', ...) for {self.bucket_name} took {dt.total_seconds()} s") return result def _open(self, name, mode="rb"): @@ -43,7 +43,7 @@ def _open(self, name, mode="rb"): result = super()._open(name, mode) if self.ietf_log_blob_timing: dt = timezone.now() - before - log(f"S3Storage timing: _open('{name}', ...) for {self.bucket_name} took {dt.total_seconds()}") + log(f"S3Storage timing: _open('{name}', ...) for {self.bucket_name} took {dt.total_seconds()} s") return result def delete(self, name): @@ -52,7 +52,7 @@ def delete(self, name): super().delete(name) if self.ietf_log_blob_timing: dt = timezone.now() - before - log(f"S3Storage timing: delete('{name}') for {self.bucket_name} took {dt.total_seconds()}") + log(f"S3Storage timing: delete('{name}') for {self.bucket_name} took {dt.total_seconds()} s") def store_file( self, From 33281c50015123ff00877a7cee614b6660bf5105 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Feb 2025 11:15:30 -0400 Subject: [PATCH 70/87] fix: deleted->null when storing a file --- ietf/doc/storage_backends.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index bab07cacd7..eb18687cab 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -60,6 +60,7 @@ def store_file( record.sha384=self.in_flight_custom_metadata[name]["sha384"] record.len=int(self.in_flight_custom_metadata[name]["len"]) record.modified=now + record.deleted = None record.save() if new_name != name: complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." From 00f0a76f8c5644d9964f60adba41827419ab6e56 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Feb 2025 11:15:59 -0400 Subject: [PATCH 71/87] style: Black --- ietf/doc/storage_backends.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index eb18687cab..d0782417da 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -14,7 +14,6 @@ from ietf.utils.timezone import timezone - # TODO-BLOBSTORE # Consider overriding save directly so that # we capture metadata for, e.g., ImageField objects @@ -44,7 +43,7 @@ def store_file( new_name = self.save(name, file) now = timezone.now() record, created = StoredObject.objects.get_or_create( - store=kind, + store=kind, name=name, defaults=dict( sha384=self.in_flight_custom_metadata[name]["sha384"], @@ -52,14 +51,14 @@ def store_file( store_created=now, created=now, modified=now, - doc_name=doc_name, # Note that these are assumed to be invariant - doc_rev=doc_rev, # for a given name - ) + doc_name=doc_name, # Note that these are assumed to be invariant + doc_rev=doc_rev, # for a given name + ), ) if not created: - record.sha384=self.in_flight_custom_metadata[name]["sha384"] - record.len=int(self.in_flight_custom_metadata[name]["len"]) - record.modified=now + record.sha384 = self.in_flight_custom_metadata[name]["sha384"] + record.len = int(self.in_flight_custom_metadata[name]["len"]) + record.modified = now record.deleted = None record.save() if new_name != name: @@ -118,7 +117,7 @@ def _get_write_parameters(self, name, content=None): params["Metadata"] = {} try: content.seek(0) - except AttributeError: # TODO-BLOBSTORE + except AttributeError: # TODO-BLOBSTORE debug.say("Encountered Non-Seekable content") raise NotImplementedError("cannot handle unseekable content") content_bytes = content.read() From 900097840770c7ed8787bb58cb705d7f58e26db9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Feb 2025 12:17:56 -0400 Subject: [PATCH 72/87] feat: log as JSON; refactor to share code; handle exceptions --- ietf/doc/storage_backends.py | 82 ++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index 63381a0201..6be35fa067 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2025, All Rights Reserved import debug # pyflakes:ignore +import json +from contextlib import contextmanager from hashlib import sha384 from io import BufferedReader from storages.backends.s3 import S3Storage @@ -14,6 +16,34 @@ from ietf.utils.timezone import timezone +@contextmanager +def maybe_log_timing(enabled, op, **kwargs): + """If enabled, log elapsed time and additional data from kwargs + + Emits log even if an exception occurs + """ + before = timezone.now() + exception = None + try: + yield + except Exception as err: + exception = err + raise + finally: + if enabled: + dt = timezone.now() - before + log( + json.dumps( + { + "log": "S3Storage_timing", + "seconds": dt.total_seconds(), + "op": op, + "exception": "" if exception is None else repr(exception), + **kwargs, + } + ) + ) + # TODO-BLOBSTORE # Consider overriding save directly so that @@ -29,30 +59,26 @@ def get_default_settings(self): return super().get_default_settings() | {"ietf_log_blob_timing": False} def _save(self, name, content): - # Only overriding this to add the option to time the operation - before = timezone.now() - result = super()._save(name, content) - if self.ietf_log_blob_timing: - dt = timezone.now() - before - log(f"S3Storage timing: _save('{name}', ...) for {self.bucket_name} took {dt.total_seconds()} s") - return result + with maybe_log_timing( + self.ietf_log_blob_timing, "_save", bucket_name=self.bucket_name, name=name + ): + return super()._save(name, content) def _open(self, name, mode="rb"): - # Only overriding this to add the option to time the operation - before = timezone.now() - result = super()._open(name, mode) - if self.ietf_log_blob_timing: - dt = timezone.now() - before - log(f"S3Storage timing: _open('{name}', ...) for {self.bucket_name} took {dt.total_seconds()} s") - return result + with maybe_log_timing( + self.ietf_log_blob_timing, + "_open", + bucket_name=self.bucket_name, + name=name, + mode=mode, + ): + return super()._open(name, mode) def delete(self, name): - # Only overriding this to add the option to time the operation - before = timezone.now() - super().delete(name) - if self.ietf_log_blob_timing: - dt = timezone.now() - before - log(f"S3Storage timing: delete('{name}') for {self.bucket_name} took {dt.total_seconds()} s") + with maybe_log_timing( + self.ietf_log_blob_timing, "delete", bucket_name=self.bucket_name, name=name + ): + super().delete(name) def store_file( self, @@ -74,7 +100,7 @@ def store_file( new_name = self.save(name, file) now = timezone.now() record, created = StoredObject.objects.get_or_create( - store=kind, + store=kind, name=name, defaults=dict( sha384=self.in_flight_custom_metadata[name]["sha384"], @@ -82,14 +108,14 @@ def store_file( store_created=now, created=now, modified=now, - doc_name=doc_name, # Note that these are assumed to be invariant - doc_rev=doc_rev, # for a given name - ) + doc_name=doc_name, # Note that these are assumed to be invariant + doc_rev=doc_rev, # for a given name + ), ) if not created: - record.sha384=self.in_flight_custom_metadata[name]["sha384"] - record.len=int(self.in_flight_custom_metadata[name]["len"]) - record.modified=now + record.sha384 = self.in_flight_custom_metadata[name]["sha384"] + record.len = int(self.in_flight_custom_metadata[name]["len"]) + record.modified = now record.save() if new_name != name: complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." @@ -147,7 +173,7 @@ def _get_write_parameters(self, name, content=None): params["Metadata"] = {} try: content.seek(0) - except AttributeError: # TODO-BLOBSTORE + except AttributeError: # TODO-BLOBSTORE debug.say("Encountered Non-Seekable content") raise NotImplementedError("cannot handle unseekable content") content_bytes = content.read() From 91824dd87142ffcaa1bd408fd436d554038ddc54 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Feb 2025 13:55:17 -0400 Subject: [PATCH 73/87] ci: add ietf_log_blob_timing option for k8s --- k8s/settings_local.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 4b73c1b1e1..9761ddf57c 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -312,6 +312,9 @@ def _multiline_to_list(s): _blob_store_bucket_prefix = os.environ.get( "DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "" ) +_blob_store_enable_profiling = ( + os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" +) try: from ietf.settings import MORE_STORAGE_NAMES except ImportError: @@ -329,5 +332,6 @@ def _multiline_to_list(s): security_token=None, client_config=boto3.session.Config(signature_version="s3v4"), bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), + ietf_log_blob_timing=_blob_store_enable_profiling, ), } From b5ac102b44e5213648b546f6833a21f6297145cd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Feb 2025 19:53:21 -0400 Subject: [PATCH 74/87] test: --no-manage-blobstore option for running tests --- ietf/utils/test_runner.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 437d201258..1c121c2394 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -723,9 +723,25 @@ def add_arguments(cls, parser): parser.add_argument('--rerun-until-failure', action='store_true', dest='rerun', default=False, help='Run the indicated tests in a loop until a failure occurs. ' ) - - def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_version_coverage=None, html_report=None, permit_mixed_migrations=None, show_logging=None, validate_html=None, validate_html_harder=None, rerun=None, **kwargs): - # + parser.add_argument('--no-manage-blobstore', action='store_false', dest='manage_blobstore', + help='Disable creating/deleting test buckets in the blob store.' + 'When this argument is used, a set of buckets with "test-" prefixed to their ' + 'names must already exist.') + + def __init__( + self, + ignore_lower_coverage=False, + skip_coverage=False, + save_version_coverage=None, + html_report=None, + permit_mixed_migrations=None, + show_logging=None, + validate_html=None, + validate_html_harder=None, + rerun=None, + manage_blobstore=True, + **kwargs + ): # self.ignore_lower_coverage = ignore_lower_coverage self.check_coverage = not skip_coverage self.save_version_coverage = save_version_coverage @@ -754,7 +770,7 @@ def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_versio # specific classes necessary to get the right ordering: self.reorder_by = (PyFlakesTestCase, MyPyTest,) + self.reorder_by + (StaticLiveServerTestCase, TemplateTagTest, CoverageTest,) #self.buckets = set() - self.blobstoremanager = TestBlobstoreManager() + self.blobstoremanager = TestBlobstoreManager() if manage_blobstore else None def setup_test_environment(self, **kwargs): global template_coverage_collection @@ -939,7 +955,8 @@ def setup_test_environment(self, **kwargs): print(" (extra pedantically)") self.vnu = start_vnu_server() - self.blobstoremanager.createTestBlobstores() + if self.blobstoremanager is not None: + self.blobstoremanager.createTestBlobstores() super(IetfTestRunner, self).setup_test_environment(**kwargs) @@ -971,7 +988,8 @@ def teardown_test_environment(self, **kwargs): if self.vnu: self.vnu.terminate() - self.blobstoremanager.destroyTestBlobstores() + if self.blobstoremanager is not None: + self.blobstoremanager.destroyTestBlobstores() super(IetfTestRunner, self).teardown_test_environment(**kwargs) From 332dafc1d65d58fe4b97c72f2a2874a1debecfaa Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Feb 2025 20:15:48 -0400 Subject: [PATCH 75/87] test: use blob store settings from env, if set --- ietf/settings_test.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 8dfab1976a..6eda581e2a 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -106,16 +106,23 @@ def tempdir_with_cleanup(**kwargs): }, } +# Configure storages for the blob store - use env settings if present. See the --no-manage-blobstore test option. +_blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000") +_blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY", "minio_root") +_blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY", "minio_pass") +_blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "test-") +_blob_store_enable_profiling = ( + os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" +) for storagename in MORE_STORAGE_NAMES: STORAGES[storagename] = { "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", "OPTIONS": dict( - endpoint_url="http://blobstore:9000", - access_key="minio_root", - secret_key="minio_pass", + endpoint_url=_blob_store_endpoint_url, + access_key=_blob_store_access_key, + secret_key=_blob_store_secret_key, security_token=None, client_config=boto3.session.Config(signature_version="s3v4"), - verify=False, bucket_name=f"test-{storagename}", ), } From 7130e3c0730b3428a5249d32427bfdc8c237281b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Feb 2025 20:18:29 -0400 Subject: [PATCH 76/87] test: actually set a couple more storage opts --- ietf/settings_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 6eda581e2a..a86c98a5fb 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -123,6 +123,7 @@ def tempdir_with_cleanup(**kwargs): secret_key=_blob_store_secret_key, security_token=None, client_config=boto3.session.Config(signature_version="s3v4"), - bucket_name=f"test-{storagename}", + bucket_name=f"{_blob_store_bucket_prefix}{storagename}", + ietf_log_blob_timing=_blob_store_enable_profiling, ), } From 34c3f5e53c706ab18dda46848b8a4c1b406a7eac Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 18 Feb 2025 10:30:29 -0600 Subject: [PATCH 77/87] feat: offswitch (#8541) * feat: offswitch * fix: apply ENABLE_BLOBSTORAGE to BlobShadowFileSystemStorage behavior --- ietf/doc/storage_utils.py | 44 ++++++++++++++++++++++++++------------- ietf/settings.py | 2 ++ ietf/utils/storage.py | 16 +++++++------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 82cf6354e3..4ce9f94c8c 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -21,13 +21,17 @@ def _get_storage(kind: str): def exists_in_storage(kind: str, name: str) -> bool: - store = _get_storage(kind) - return store.exists_in_storage(kind, name) + if settings.ENABLE_BLOBSTORAGE: + store = _get_storage(kind) + return store.exists_in_storage(kind, name) + else: + return False def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None: - store = _get_storage(kind) - store.remove_from_storage(kind, name, warn_if_missing) + if settings.ENABLE_BLOBSTORAGE: + store = _get_storage(kind) + store.remove_from_storage(kind, name, warn_if_missing) return None @@ -41,8 +45,9 @@ def store_file( doc_rev: Optional[str] = None, ) -> None: # debug.show('f"asked to store {name} into {kind}"') - store = _get_storage(kind) - store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev) + if settings.ENABLE_BLOBSTORAGE: + store = _get_storage(kind) + store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev) return None @@ -54,7 +59,9 @@ def store_bytes( doc_name: Optional[str] = None, doc_rev: Optional[str] = None, ) -> None: - return store_file(kind, name, ContentFile(content), allow_overwrite) + if settings.ENABLE_BLOBSTORAGE: + store_file(kind, name, ContentFile(content), allow_overwrite) + return None def store_str( @@ -65,18 +72,25 @@ def store_str( doc_name: Optional[str] = None, doc_rev: Optional[str] = None, ) -> None: - content_bytes = content.encode("utf-8") - return store_bytes(kind, name, content_bytes, allow_overwrite) + if settings.ENABLE_BLOBSTORAGE: + content_bytes = content.encode("utf-8") + store_bytes(kind, name, content_bytes, allow_overwrite) + return None def retrieve_bytes(kind: str, name: str) -> bytes: - store = _get_storage(kind) - with store.open(name) as f: - content = f.read() + content = b"" + if settings.ENABLE_BLOBSTORAGE: + store = _get_storage(kind) + with store.open(name) as f: + content = f.read() return content def retrieve_str(kind: str, name: str) -> str: - content_bytes = retrieve_bytes(kind, name) - # TODO: try to decode all the different ways doc.text() does - return content_bytes.decode("utf-8") + content = "" + if settings.ENABLE_BLOBSTORAGE: + content_bytes = retrieve_bytes(kind, name) + # TODO-BLOBSTORE: try to decode all the different ways doc.text() does + content = content_bytes.decode("utf-8") + return content diff --git a/ietf/settings.py b/ietf/settings.py index 3634749ea0..256603a46d 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -184,6 +184,8 @@ # Server-side static.ietf.org URL (used in pdfized) STATIC_IETF_ORG_INTERNAL = STATIC_IETF_ORG +ENABLE_BLOBSTORAGE = True + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py index bd8324232c..9f41f3d50f 100644 --- a/ietf/utils/storage.py +++ b/ietf/utils/storage.py @@ -2,6 +2,7 @@ """Django Storage classes""" from pathlib import Path +from django.conf import settings from django.core.files.storage import FileSystemStorage from ietf.doc.storage_utils import store_file from .log import log @@ -39,13 +40,14 @@ def save(self, name, content, max_length=None): # Write content to the filesystem - this deals with chunks, etc... saved_name = super().save(name, content, max_length) - # Retrieve the content and write to the blob store - blob_name = Path(saved_name).name # strips path - try: - with self.open(saved_name, "rb") as f: - store_file(self.kind, blob_name, f, allow_overwrite=True) - except Exception as err: - log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}") + if settings.ENABLE_BLOBSTORAGE: + # Retrieve the content and write to the blob store + blob_name = Path(saved_name).name # strips path + try: + with self.open(saved_name, "rb") as f: + store_file(self.kind, blob_name, f, allow_overwrite=True) + except Exception as err: + log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}") return saved_name # includes the path! def deconstruct(self): From 95d8455841d9a5ff77c16a6fdb3aaf402a3d4753 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 13:06:05 -0400 Subject: [PATCH 78/87] chore: log timing of blob reads --- ietf/doc/storage_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 4ce9f94c8c..4f0516339a 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -79,11 +79,18 @@ def store_str( def retrieve_bytes(kind: str, name: str) -> bytes: + from ietf.doc.storage_backends import maybe_log_timing content = b"" if settings.ENABLE_BLOBSTORAGE: store = _get_storage(kind) with store.open(name) as f: - content = f.read() + with maybe_log_timing( + hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, + "read", + bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", + name=name, + ): + content = f.read() return content From e1969b715cf93756eac2d33e91fb5b403385bfea Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 16:11:11 -0400 Subject: [PATCH 79/87] chore: import Config from botocore.config --- dev/deploy-to-container/settings_local.py | 5 +++-- dev/diff/settings_local.py | 5 +++-- dev/tests/settings_local.py | 5 +++-- docker/configs/settings_local.py | 5 +++-- docker/scripts/app-configure-blobstore.py | 2 +- ietf/settings.py | 1 - ietf/settings_test.py | 5 +++-- ietf/utils/test_runner.py | 9 ++++++--- k8s/settings_local.py | 5 +++-- 9 files changed, 25 insertions(+), 17 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index df0e7cc7f1..b7c7212cf4 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES +import botocore.config ALLOWED_HOSTS = ['*'] @@ -89,7 +90,7 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=botocore.config.Config(signature_version="s3v4"), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index b0994dcfbf..8940d0b87f 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES +import botocore.config ALLOWED_HOSTS = ['*'] @@ -76,7 +77,7 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=botocore.config.Config(signature_version="s3v4"), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index b8815b837b..aa4fb6550a 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES +import botocore.config ALLOWED_HOSTS = ['*'] @@ -75,7 +76,7 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=botocore.config.Config(signature_version="s3v4"), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 8771053419..7f3d180737 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import boto3, STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES +import botocore.config ALLOWED_HOSTS = ['*'] @@ -46,7 +47,7 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=botocore.config.Config(signature_version="s3v4"), verify=False, bucket_name=storagename, ), diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index c8d5f40556..7b5ce962eb 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -15,7 +15,7 @@ def init_blobstore(): aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"), aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"), aws_session_token=None, - config=boto3.session.Config(signature_version="s3v4"), + config=botocore.config.Config(signature_version="s3v4"), verify=False, ) for bucketname in MORE_STORAGE_NAMES: diff --git a/ietf/settings.py b/ietf/settings.py index 256603a46d..9d0ab26386 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -6,7 +6,6 @@ # BASE_DIR and "settings_local" are from # http://code.djangoproject.com/wiki/SplitSettings -import boto3 # pyflakes:ignore import os import sys import datetime diff --git a/ietf/settings_test.py b/ietf/settings_test.py index a86c98a5fb..9240f72ccc 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,8 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import boto3, STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES +import botocore.config import debug # pyflakes:ignore debug.debug = True @@ -122,7 +123,7 @@ def tempdir_with_cleanup(**kwargs): access_key=_blob_store_access_key, secret_key=_blob_store_secret_key, security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=botocore.config.Config(signature_version="s3v4"), bucket_name=f"{_blob_store_bucket_prefix}{storagename}", ietf_log_blob_timing=_blob_store_enable_profiling, ), diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 1c121c2394..3c89a2d01c 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -49,6 +49,7 @@ import tempfile import copy import boto3 +import botocore.config import factory.random import urllib3 import warnings @@ -86,6 +87,8 @@ from ietf.utils.test_smtpserver import SMTPTestServerDriver from ietf.utils.test_utils import TestCase +from mypy_boto3_s3.service_resource import Bucket + loaded_templates = set() visited_urls = set() @@ -1248,15 +1251,15 @@ def tearDown(self): class TestBlobstoreManager(): # N.B. buckets and blobstore are intentional Class-level attributes - buckets = set() + buckets: set[Bucket] = set() blobstore = boto3.resource("s3", endpoint_url="http://blobstore:9000", aws_access_key_id="minio_root", aws_secret_access_key="minio_pass", aws_session_token=None, - config=boto3.session.Config(signature_version="s3v4"), - #config=boto3.session.Config(signature_version=botocore.UNSIGNED), + config = botocore.config.Config(signature_version="s3v4"), + #config=botocore.config.Config(signature_version=botocore.UNSIGNED), verify=False ) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 9761ddf57c..7728459aa5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -320,7 +320,8 @@ def _multiline_to_list(s): except ImportError: pass # Don't fail if MORE_STORAGE_NAMES is not there, just don't configure it else: - from ietf.settings import boto3, STORAGES # do fail if these aren't found! + from ietf.settings import STORAGES # do fail if these aren't found! + import botocore.config for storage_name in MORE_STORAGE_NAMES: STORAGES[storage_name] = { @@ -330,7 +331,7 @@ def _multiline_to_list(s): access_key=_blob_store_access_key, secret_key=_blob_store_secret_key, security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=botocore.config.Config(signature_version="s3v4"), bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), ietf_log_blob_timing=_blob_store_enable_profiling, ), From 39ddc71b79679775e2e4738d2db3a56f11e74d08 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 16:12:04 -0400 Subject: [PATCH 80/87] chore(deps): import boto3-stubs / botocore botocore is implicitly imported, but make it explicit since we refer to it directly --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 093fa25705..66a785e929 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,8 @@ bibtexparser>=1.2.0 # Only used in tests bleach>=6 types-bleach>=6 boto3>=1.35,<1.36 +boto3-stubs[s3]>=1.35,<1.36 +botocore>=1.35,<1.36 celery>=5.2.6 coverage>=4.5.4,<5.0 # Coverage 5.x moves from a json database to SQLite. Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views defusedxml>=0.7.1 # for TastyPie when using xml; not a declared dependency From b0511af9f9f2802407c17be29861fecc1cddeff6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 16:12:37 -0400 Subject: [PATCH 81/87] chore: drop type annotation that mypy loudly ignores --- ietf/doc/storage_backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py index aa6edcd139..5eeab040e5 100644 --- a/ietf/doc/storage_backends.py +++ b/ietf/doc/storage_backends.py @@ -7,7 +7,7 @@ from hashlib import sha384 from io import BufferedReader from storages.backends.s3 import S3Storage -from typing import Dict, Optional, Union +from typing import Optional, Union from django.core.files.base import File @@ -51,7 +51,7 @@ def maybe_log_timing(enabled, op, **kwargs): class CustomS3Storage(S3Storage): def __init__(self, **settings): - self.in_flight_custom_metadata: Dict[str, Dict[str, str]] = {} + self.in_flight_custom_metadata = {} # type is Dict[str, Dict[str, str]] super().__init__(**settings) def get_default_settings(self): From 0642c66dd9a0f6204a8f3a07cec306bbace638e0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 16:14:16 -0400 Subject: [PATCH 82/87] refactor: add storage methods via mixin Shares code between Document and DocHistory without putting it in the base DocumentInfo class, which lacks the name field. Also makes mypy happy. --- ietf/doc/models.py | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 840eafd38a..55da70972c 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -12,7 +12,7 @@ from io import BufferedReader from pathlib import Path from lxml import etree -from typing import Optional, TYPE_CHECKING, Union +from typing import Optional, Protocol, TYPE_CHECKING, Union from weasyprint import HTML as wpHTML from weasyprint.text.fonts import FontConfiguration @@ -722,23 +722,50 @@ def referenced_by_rfcs_as_rfc_or_draft(self): refs_to |= self.came_from_draft().referenced_by_rfcs() return refs_to + class Meta: + abstract = True + + +class HasNameRevAndTypeIdProtocol(Protocol): + """Typing Protocol describing a class that has name, rev, and type_id properties""" + @property + def name(self) -> str: ... + @property + def rev(self) -> str: ... + @property + def type_id(self) -> str: ... + + +class StorableMixin: + """Mixin that adds storage helpers to a DocumentInfo subclass""" def store_str( - self, name: str, content: str, allow_overwrite: bool = False + self: HasNameRevAndTypeIdProtocol, + name: str, + content: str, + allow_overwrite: bool = False ) -> None: return utils_store_str(self.type_id, name, content, allow_overwrite, self.name, self.rev) - + def store_bytes( - self, name: str, content: bytes, allow_overwrite: bool = False, doc_name: Optional[str] = None, doc_rev: Optional[str] = None + self: HasNameRevAndTypeIdProtocol, + name: str, + content: bytes, + allow_overwrite: bool = False, + doc_name: Optional[str] = None, + doc_rev: Optional[str] = None ) -> None: return utils_store_bytes(self.type_id, name, content, allow_overwrite, self.name, self.rev) def store_file( - self, name: str, file: Union[File,BufferedReader], allow_overwrite: bool = False, doc_name: Optional[str] = None, doc_rev: Optional[str] = None + self: HasNameRevAndTypeIdProtocol, + name: str, + file: Union[File, BufferedReader], + allow_overwrite: bool = False, + doc_name: Optional[str] = None, + doc_rev: Optional[str] = None ) -> None: return utils_store_file(self.type_id, name, file, allow_overwrite, self.name, self.rev) - class Meta: - abstract = True STATUSCHANGE_RELATIONS = ('tops','tois','tohist','toinf','tobcp','toexp') @@ -892,7 +919,7 @@ def role_for_doc(self): 'invalid' ) -class Document(DocumentInfo): +class Document(StorableMixin, DocumentInfo): name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True) @@ -1214,7 +1241,7 @@ class DocHistoryAuthor(DocumentAuthorInfo): def __str__(self): return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order) -class DocHistory(DocumentInfo): +class DocHistory(StorableMixin, DocumentInfo): doc = ForeignKey(Document, related_name="history_set") name = models.CharField(max_length=255) From cadc4667c8b25ef98ce029081d77e9845dcd1a1f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 16:56:31 -0400 Subject: [PATCH 83/87] feat: add timeout / retry limit to boto client --- dev/deploy-to-container/settings_local.py | 7 ++++++- dev/diff/settings_local.py | 7 ++++++- dev/tests/settings_local.py | 7 ++++++- docker/configs/settings_local.py | 7 ++++++- ietf/settings.py | 4 ++++ ietf/settings_test.py | 7 ++++++- k8s/settings_local.py | 7 ++++++- 7 files changed, 40 insertions(+), 6 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index df0e7cc7f1..1e105dd721 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -89,7 +89,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index b0994dcfbf..c8e327115e 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -76,7 +76,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index b8815b837b..72caad60a9 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -75,7 +75,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=f"test-{storagename}", ), diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 8771053419..019caeba9c 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -46,7 +46,12 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), verify=False, bucket_name=storagename, ), diff --git a/ietf/settings.py b/ietf/settings.py index 256603a46d..83c408e656 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -186,6 +186,10 @@ ENABLE_BLOBSTORAGE = True +BLOBSTORAGE_MAX_ATTEMPTS = 1 +BLOBSTORAGE_CONNECT_TIMEOUT = 2 +BLOBSTORAGE_READ_TIMEOUT = 2 + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index a86c98a5fb..ba258570bf 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -122,7 +122,12 @@ def tempdir_with_cleanup(**kwargs): access_key=_blob_store_access_key, secret_key=_blob_store_secret_key, security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), bucket_name=f"{_blob_store_bucket_prefix}{storagename}", ietf_log_blob_timing=_blob_store_enable_profiling, ), diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 9761ddf57c..42d4e0c3ce 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -330,7 +330,12 @@ def _multiline_to_list(s): access_key=_blob_store_access_key, secret_key=_blob_store_secret_key, security_token=None, - client_config=boto3.session.Config(signature_version="s3v4"), + client_config=boto3.session.Config( + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), ietf_log_blob_timing=_blob_store_enable_profiling, ), From 9371a5f52bb2724ab43423e9c9d66d56d0f88588 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 17:00:42 -0400 Subject: [PATCH 84/87] ci: let k8s config the timeouts via env --- k8s/settings_local.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 42d4e0c3ce..1ecd77a5d5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -315,6 +315,15 @@ def _multiline_to_list(s): _blob_store_enable_profiling = ( os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" ) +_blob_store_max_attempts = ( + os.environ.get("DATATRACKER_BLOB_STORE_MAX_ATTEMPTS", BLOBSTORAGE_MAX_ATTEMPTS) +) +_blob_store_connect_timeout = ( + os.environ.get("DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT) +) +_blob_store_read_timeout = ( + os.environ.get("DATATRACKER_BLOB_STORE_READ_TIMEOUT", BLOBSTORAGE_READ_TIMEOUT) +) try: from ietf.settings import MORE_STORAGE_NAMES except ImportError: @@ -332,9 +341,9 @@ def _multiline_to_list(s): security_token=None, client_config=boto3.session.Config( signature_version="s3v4", - connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, - read_timeout=BLOBSTORAGE_READ_TIMEOUT, - retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + connect_timeout=_blob_store_connect_timeout, + read_timeout=_blob_store_read_timeout, + retries={"total_max_attempts": _blob_store_max_attempts}, ), bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), ietf_log_blob_timing=_blob_store_enable_profiling, From 15e2c94c82b9c77a5f756e7be01cb0442bd7f585 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 19 Feb 2025 16:03:39 -0600 Subject: [PATCH 85/87] chore: repair merge resolution typo --- docker/configs/settings_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 73a625ea3c..19e1a56620 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -47,7 +47,7 @@ access_key="minio_root", secret_key="minio_pass", security_token=None, - client_config=botocore.confg.Config( + client_config=botocore.config.Config( signature_version="s3v4", connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, read_timeout=BLOBSTORAGE_READ_TIMEOUT, From 8d4308b4c31148fab550a1d328fc3f0e60985e7a Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 19 Feb 2025 16:19:36 -0600 Subject: [PATCH 86/87] chore: tweak settings imports --- dev/deploy-to-container/settings_local.py | 2 +- dev/diff/settings_local.py | 2 +- dev/tests/settings_local.py | 2 +- docker/configs/settings_local.py | 2 +- ietf/settings_test.py | 2 +- k8s/settings_local.py | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index aaee07f0ea..e878206bd5 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS import botocore.config ALLOWED_HOSTS = ['*'] diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index c52ee6d8ab..9e0806a8a6 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS import botocore.config ALLOWED_HOSTS = ['*'] diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index db1676d484..f2166053a7 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS import botocore.config ALLOWED_HOSTS = ['*'] diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 19e1a56620..46833451c1 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS import botocore.config ALLOWED_HOSTS = ['*'] diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 9cdb435097..fe77152d42 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES +from ietf.settings import STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS import botocore.config import debug # pyflakes:ignore debug.debug = True diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 8f89894d1c..0ef2593b7e 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -7,6 +7,7 @@ from ietf import __release_hash__ from ietf.settings import * # pyflakes:ignore +from ietf.settings import BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS def _multiline_to_list(s): From a661280a7e9256b62f95859c994de21893af9541 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Feb 2025 19:27:50 -0400 Subject: [PATCH 87/87] chore: simplify k8s/settings_local.py imports --- k8s/settings_local.py | 45 ++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 0ef2593b7e..912607f466 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -7,7 +7,8 @@ from ietf import __release_hash__ from ietf.settings import * # pyflakes:ignore -from ietf.settings import BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS +from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS +import botocore.config def _multiline_to_list(s): @@ -325,29 +326,21 @@ def _multiline_to_list(s): _blob_store_read_timeout = ( os.environ.get("DATATRACKER_BLOB_STORE_READ_TIMEOUT", BLOBSTORAGE_READ_TIMEOUT) ) -try: - from ietf.settings import MORE_STORAGE_NAMES -except ImportError: - pass # Don't fail if MORE_STORAGE_NAMES is not there, just don't configure it -else: - from ietf.settings import STORAGES # do fail if these aren't found! - import botocore.config - - for storage_name in MORE_STORAGE_NAMES: - STORAGES[storage_name] = { - "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", - "OPTIONS": dict( - endpoint_url=_blob_store_endpoint_url, - access_key=_blob_store_access_key, - secret_key=_blob_store_secret_key, - security_token=None, - client_config=botocore.config.Config( - signature_version="s3v4", - connect_timeout=_blob_store_connect_timeout, - read_timeout=_blob_store_read_timeout, - retries={"total_max_attempts": _blob_store_max_attempts}, - ), - bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), - ietf_log_blob_timing=_blob_store_enable_profiling, +for storage_name in MORE_STORAGE_NAMES: + STORAGES[storage_name] = { + "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", + "OPTIONS": dict( + endpoint_url=_blob_store_endpoint_url, + access_key=_blob_store_access_key, + secret_key=_blob_store_secret_key, + security_token=None, + client_config=botocore.config.Config( + signature_version="s3v4", + connect_timeout=_blob_store_connect_timeout, + read_timeout=_blob_store_read_timeout, + retries={"total_max_attempts": _blob_store_max_attempts}, ), - } + bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), + ietf_log_blob_timing=_blob_store_enable_profiling, + ), + }