SciTools · pp-mo · Jun 9, 2025 · Jun 6, 2025 · Jun 6, 2025 · Jun 9, 2025
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -6,4 +6,4 @@
 #  and to reinforce the culture of using this label.
 benchmark_this:
 - changed-files:
-  - any-glob-to-any-file: ['requirements/locks/*.lock', "setup.py"]
+  - any-glob-to-any-file: ['requirements/locks/*.lock']
diff --git a/.github/workflows/benchmarks_run.yml b/.github/workflows/benchmarks_run.yml
@@ -36,7 +36,7 @@ jobs:
         uses: marceloprado/has-changed-path@df1b7a3161b8fb9fd8c90403c66a9e66dfde50cb
         with:
           # SEE ALSO .github/labeler.yml .
-          paths: requirements/locks/*.lock setup.py
+          paths: requirements/locks/*.lock
       - id: overnight
         name: Check overnight scenario
         if: github.event_name != 'pull_request'

diff --git a/.github/workflows/benchmarks_validate.yml b/.github/workflows/benchmarks_validate.yml
@@ -0,0 +1,49 @@
+name: benchmarks-validate
+run-name: Validate the benchmarking setup
+
+on:
+  push:
+    branches:
+      - "main"
+      - "v*x"
+    tags:
+      - "v*"
+  pull_request:
+    branches:
+      - "*"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+
+    env:
+      # Lets us manually bump the cache to rebuild
+      ENV_CACHE_BUILD: "0"
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Nox
+        run: |
+          pip install nox!=2025.05.01
+
+      - name: Cache environment directories
+        id: cache-env-dir
+        uses: actions/cache@v4
+        with:
+          path: |
+            .nox
+            benchmarks/.asv/env
+            $CONDA/pkgs
+          key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }}
+
+      - name: Validate setup
+        run: nox -s benchmarks -- validate
diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py
@@ -6,6 +6,11 @@
 
 from os import environ
 
+import iris
+
+from . import generate_data
+from .generate_data.um_files import create_um_files
+
 
 def disable_repeat_between_setup(benchmark_object):
     """Benchmark where object persistence would be inappropriate (decorator).
@@ -50,3 +55,22 @@ def on_demand_benchmark(benchmark_object):
     """
     if "ON_DEMAND_BENCHMARKS" in environ:
         return benchmark_object
+
+
+@on_demand_benchmark
+class ValidateSetup:
+    """Simple benchmarks that exercise all elements of our setup."""
+
+    params = [1, 2]
+
+    def setup(self, param):
+        generate_data.REUSE_DATA = False
+        (self.file_path,) = create_um_files(
+            param, param, param, param, False, ["NetCDF"]
+        ).values()
+
+    def time_validate(self, param):
+        _ = iris.load(self.file_path)
+
+    def tracemalloc_validate(self, param):
+        _ = iris.load(self.file_path)
diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py
@@ -619,6 +619,50 @@ def func(args: argparse.Namespace) -> None:
         _subprocess_runner(asv_command, asv=True)
 
 
+class Validate(_SubParserGenerator):
+    name = "validate"
+    description = (
+        "Quickly check that the benchmark architecture works as intended with "
+        "the current codebase. Things that are checked: env creation/update, "
+        "package build/install/uninstall, artificial data creation."
+    )
+    epilog = "Sole acceptable syntax: python bm_runner.py validate"
+
+    @staticmethod
+    def func(args: argparse.Namespace) -> None:
+        _setup_common()
+
+        git_command = shlex.split("git rev-parse HEAD")
+        head_sha = _subprocess_runner_capture(git_command)[:8]
+
+        # Find the most recent commit where the lock-files are not
+        #  identical to HEAD - will force environment updates.
+        locks_dir = Path(__file__).parents[1] / "requirements" / "locks"
+        assert locks_dir.is_dir()
+        git_command = shlex.split(
+            f"git log -1 --pretty=format:%P -- {locks_dir.resolve()}"
+        )
+        locks_sha = _subprocess_runner_capture(git_command)[:8]
+
+        with NamedTemporaryFile("w") as hashfile:
+            hashfile.writelines([locks_sha, "\n", head_sha])
+            hashfile.flush()
+            asv_command = shlex.split(
+                f"run HASHFILE:{hashfile.name} --bench ValidateSetup "
+                "--attribute rounds=1 --show-stderr"
+            )
+            extra_env = environ | {"ON_DEMAND_BENCHMARKS": "1"}
+            _subprocess_runner(asv_command, asv=True, env=extra_env)
+
+    # No arguments permitted for this subclass:
+
+    def add_arguments(self) -> None:
+        pass
+
+    def add_asv_arguments(self) -> None:
+        pass
+
+
 class GhPost(_SubParserGenerator):
     name = "_gh_post"
     description = (
@@ -670,6 +714,7 @@ def main():
         SPerf,
         Custom,
         TrialRun,
+        Validate,
         GhPost,
     )
 

diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst
@@ -92,6 +92,9 @@ This document explains the changes made to Iris for this release
    on `lychee <https://github.com/lycheeverse/lychee-action>`__.
    (:issue:`4140`, :pull:`6386`)
 
+#. `@trexfeathers`_ added a CI workflow to quickly validate that the
+   benchmarking setup is still working. (:pull:`6496`)
+
 
 .. comment
     Whatsnew author names (@github name) in alphabetical order. Note that,