anchore · wagoodman · Feb 18, 2025 · Feb 20, 2025
diff --git a/.github/workflows/daily-db-publisher-r2.yaml b/.github/workflows/daily-db-publisher-r2.yaml
@@ -29,7 +29,7 @@ jobs:
     # a) booleans come across as string types :(
     # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
     if: ${{ github.event.inputs.publish-databases != 'false' }}
-    name: "Pull vulnerability data"
+    name: "Discover schemas"
     runs-on: ubuntu-24.04
     outputs:
       schema-versions: ${{ steps.read-schema-versions.outputs.schema-versions }}
@@ -53,7 +53,7 @@ jobs:
     # a) booleans come across as string types :(
     # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
     if: ${{ github.event.inputs.publish-databases != 'false' }}
-    name: "Generate and publish DBs"
+    name: "Publish"
     needs: discover-schema-versions
     runs-on: ubuntu-22.04-4core-16gb
     strategy:
@@ -103,54 +103,9 @@ jobs:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
         if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}
 
-  publish-listing-file:
-    # fun! https://github.com/actions/runner/issues/491#issuecomment-850884422
-    # essentially even if the workflow dispatch job is skipping steps, we still want to run this step.
-    # however, if not running from a workflow dispatch then we want the job ordering to be honored.
-    # also...
-    # note about workflow dispatch inputs and booleans:
-    # a) booleans come across as string types :(
-    # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
-    if: |
-      always() &&
-      (needs.generate-and-publish-dbs.result == 'success' || needs.generate-and-publish-dbs.result == 'skipped') &&
-      github.event.inputs.publish-listing != 'false'
-
-    name: "Publish listing file"
-    needs: generate-and-publish-dbs
-    runs-on: ubuntu-22.04-4core-16gb
-    steps:
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
-        with:
-          submodules: true
-
-      - name: Bootstrap environment
-        uses: ./.github/actions/bootstrap
-
-      - name: Publish listing file
-        run: |
-          uv run \
-            grype-db-manager \
-              -c ./config/grype-db-manager/publish-production-r2.yaml \
-                listing update
-        env:
-            AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_CLOUDFLARE_ACCESS_KEY_ID }}
-            AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_CLOUDFLARE_SECRET_ACCESS_KEY }}
-            GRYPE_DB_MANAGER_DISTRIBUTION_S3_ENDPOINT_URL: ${{ secrets.TOOLBOX_CLOUDFLARE_R2_ENDPOINT }}
-
-      - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2
-        with:
-          status: ${{ job.status }}
-          fields: workflow,eventName,job
-          text: Publishing the Grype DB listing file has failed
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
-        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}
-
   sync-listing-file-to-s3:
     name: "Sync listing file to S3"
     needs:
-      - publish-listing-file
+      - generate-and-publish-dbs
     uses: ./.github/workflows/copy-listing-from-r2.yaml
     secrets: inherit
diff --git a/.github/workflows/fix-v5-listing.yaml b/.github/workflows/fix-v5-listing.yaml
@@ -0,0 +1,65 @@
+name: 'Fix v5 listing'
+on:
+  # allow for kicking off DB builds manually
+  workflow_dispatch:
+    inputs:
+      publish-databases:
+        description: "build new databases and upload to S3"
+        type: boolean
+        required: true
+        default: true
+      publish-listing:
+        description: "use S3 state to update and publish listing file"
+        type: boolean
+        required: true
+        default: true
+
+  # run 4 AM (UTC) daily
+  schedule:
+    - cron:  '0 4 * * *'
+
+env:
+  CGO_ENABLED: "0"
+  SLACK_NOTIFICATIONS: true
+  FORCE_COLOR: true
+
+jobs:
+
+  publish-listing-file:
+    name: "Publish listing file"
+    runs-on: ubuntu-22.04-4core-16gb
+    steps:
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
+        with:
+          submodules: true
+
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+
+      - name: Publish listing file
+        run: |
+          uv run \
+            grype-db-manager \
+              -c ./config/grype-db-manager/publish-production-r2.yaml \
+                listing update
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_CLOUDFLARE_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_CLOUDFLARE_SECRET_ACCESS_KEY }}
+          GRYPE_DB_MANAGER_DISTRIBUTION_S3_ENDPOINT_URL: ${{ secrets.TOOLBOX_CLOUDFLARE_R2_ENDPOINT }}
+
+      - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2
+        with:
+          status: ${{ job.status }}
+          fields: workflow,eventName,job
+          text: Publishing the Grype DB listing file has failed
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
+        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}
+
+  sync-listing-file-to-s3:
+    name: "Sync listing file to S3"
+    needs:
+      - publish-listing-file
+    uses: ./.github/workflows/copy-listing-from-r2.yaml
+    secrets: inherit
diff --git a/manager/src/grype_db_manager/cli/db.py b/manager/src/grype_db_manager/cli/db.py
@@ -9,7 +9,7 @@
 from yardstick.cli.validate import validate as yardstick_validate
 
 from grype_db_manager import db, grypedb, s3utils
-from grype_db_manager.cli import config, error
+from grype_db_manager.cli import config, error, listing
 from grype_db_manager.db.format import Format
 from grype_db_manager.grypedb import DB_DIR, DBManager, GrypeDB
 
@@ -280,12 +280,13 @@ def _validate_latest(cfg: config.Application, latest_file: str, archive_path: st
     )
 
 
-@group.command(name="upload", help="upload a grype database")
+@group.command(name="upload", help="upload a grype database and update listing")
 @click.option("--ttl-seconds", "-t", default=DEFAULT_TTL_SECONDS, help="the TTL for the uploaded DB (should be relatively high)")
 @click.argument("db-uuid")
 @click.pass_obj
+@click.pass_context
 @error.handle_exception(handle=(ValueError, s3utils.CredentialsError))
-def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None:
+def upload_db(ctx: click.core.Context, cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None:
     if cfg.assert_aws_credentials:
         s3utils.check_credentials()
 
@@ -324,6 +325,8 @@ def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None:
         )
 
         click.echo(f"DB latest.json {db_uuid!r} uploaded to s3://{s3_bucket}/{s3_path}")
+    else:
+        ctx.invoke(listing.update_listing)
 
 
 @group.command(name="build-and-upload", help="upload a grype database")

diff --git a/manager/tests/cli/test_legacy_workflows.py b/manager/tests/cli/test_legacy_workflows.py
@@ -200,14 +200,11 @@ def test_workflow_4(cli_env, command, logger, tmp_path, grype):
     )
     assert "Quality gate passed!" in stdout
     assert "' uploaded to s3://testbucket/grype/databases" in stdout
-
-    logger.step("case 2: update the listing file based on the DB uploaded")
-
-    # update the listing file and validate
-    stdout, _ = command.run("grype-db-manager -v listing update", env=cli_env)
     assert "Validation passed" in stdout
     assert "listing.json uploaded to s3://testbucket/grype/databases" in stdout
 
+    logger.step("case 2: update the listing file based on the DB uploaded")
+
     # set grype environment variables
     cli_env.update(
         {

diff --git a/manager/tests/unit/cli/test_db.py b/manager/tests/unit/cli/test_db.py
@@ -20,18 +20,24 @@ def test_upload_db(mocker, test_dir_path, redact_aws_credentials):
         archive_path="some/path/to/archive.tar.gz",
     )
 
-    s3_mock = mocker.patch("grype_db_manager.cli.db.s3utils")
-    s3_mock.upload_file.return_value = None
+    s3_db_mock = mocker.patch("grype_db_manager.cli.db.s3utils")
+    s3_db_mock.upload_file.return_value = None
+
+    listing_mock = mocker.patch("grype_db_manager.cli.listing.update_listing")
+    listing_mock.return_value = None
 
     runner = CliRunner()
     result = runner.invoke(cli.cli, f"-c {config_path} db upload some-db-uuid".split())
 
     assert result.exit_code == 0
 
     # ensure the s3 mock was called with the right arguments
-    s3_mock.upload_file.assert_called_once_with(
+    s3_db_mock.upload_file.assert_called_once_with(
         path="some/path/to/archive.tar.gz",
         bucket="testbucket",
         key="grype/databases/archive.tar.gz",
         CacheControl="public,max-age=31536000",
     )
+
+    # ensure the listing mock was called once
+    listing_mock.assert_called_once()
diff --git a/uv.lock b/uv.lock