Introduce scenario-based regression testing framework with expanded configuration coverage

.github/workflows/daily.yaml

@@ -11,14 +11,16 @@ concurrency:
 
 jobs:
   unit:
-    name: Unit (${{ matrix.os }}, ${{ matrix.python-version }})
+    name: Unit (${{ matrix.os }}, py${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
+
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04, macos-15, windows-2025]
         python-version: ["3.12", "3.13", "3.14"]
+
     steps:
       - name: Checkout
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
@@ -29,10 +31,10 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: pip
 
-      - name: Install (testing)
+      - name: Install testing dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[testing]
 
-      - name: Pytest (unit) • ${{ matrix.os }} • py${{ matrix.python-version }}
+      - name: Run unit tests
         run: python -m pytest tests/unit

.github/workflows/pr.yaml

@@ -17,12 +17,13 @@ jobs:
       matrix:
         os: [ubuntu-24.04, macos-15, windows-2025]
         python-version: ["3.12", "3.13", "3.14"]
+
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
@@ -35,17 +36,50 @@ jobs:
       - name: Pytest (unit) • ${{ matrix.os }}, ${{ matrix.python-version }}
         run: python -m pytest tests/unit
 
+  discover-systems:
+    name: Discover regression systems
+    runs-on: ubuntu-24.04
+    outputs:
+      systems: ${{ steps.set-systems.outputs.systems }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.14"
+          cache: pip
+
+      - name: Install testing dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .[testing]
+
+      - name: Discover systems
+        id: set-systems
+        run: |
+          SYSTEMS=$(python -m tests.regression.list_systems)
+          echo "systems=$SYSTEMS" >> $GITHUB_OUTPUT
+
   regression-quick:
-    name: Regression (quick)
-    needs: unit
+    name: Regression (fast) • ${{ matrix.system }}
+    needs: [unit, discover-systems]
     runs-on: ubuntu-24.04
     timeout-minutes: 35
+
+    strategy:
+      fail-fast: false
+      matrix:
+        system: ${{ fromJson(needs.discover-systems.outputs.systems) }}
+
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Python 3.14
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: "3.14"
           cache: pip
@@ -54,28 +88,31 @@ jobs:
         uses: actions/cache@v4
         with:
           path: .testdata
-          key: codeentropy-testdata-v1-${{ runner.os }}-py3.14
+          key: codeentropy-testdata-${{ runner.os }}-py314
 
-      - name: Install (testing)
+      - name: Install testing dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[testing]
 
-      - name: Pytest (regression quick)
-        run: python -m pytest tests/regression
+      - name: Run fast regression tests (per system)
+        run: |
+          python -m pytest tests/regression \
+            -m "not slow" \
+            -n auto \
+            --dist=loadscope \
+            -k "${{ matrix.system }}" \
+            -vv \
+            --durations=20
 
       - name: Upload artifacts (failure)
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: quick-regression-failure
+          name: quick-regression-failure-${{ matrix.system }}
           path: |
             .testdata/**
-            tests/regression/**/.pytest_cache/**
-            /tmp/pytest-of-*/pytest-*/**/config.yaml
-            /tmp/pytest-of-*/pytest-*/**/codeentropy_stdout.txt
-            /tmp/pytest-of-*/pytest-*/**/codeentropy_stderr.txt
-            /tmp/pytest-of-*/pytest-*/**/codeentropy_output.json
+            /tmp/pytest-of-*/pytest-*/**
 
   docs:
     name: Docs
@@ -93,7 +130,7 @@ jobs:
           python-version: "3.14"
           cache: pip
 
-      - name: Install
+      - name: Install docs dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[docs]
@@ -114,23 +151,23 @@ jobs:
     name: Pre-commit
     runs-on: ubuntu-24.04
     timeout-minutes: 15
+
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Python 3.14
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: "3.14"
           cache: pip
 
-      - name: Install (pre-commit)
+      - name: Install pre-commit dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[pre-commit]
 
       - name: Run pre-commit
-        shell: bash
         run: |
           pre-commit install
           pre-commit run --all-files || {
@@ -144,8 +181,9 @@ jobs:
     needs: unit
     runs-on: ubuntu-24.04
     timeout-minutes: 30
+
     steps:
-      - name: Checkout repo
+      - name: Checkout
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Python 3.14
@@ -154,20 +192,20 @@ jobs:
           python-version: "3.14"
           cache: pip
 
-      - name: Install (testing)
+      - name: Install testing dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[testing]
 
-      - name: Run unit test suite with coverage
+      - name: Run coverage
         run: |
           pytest tests/unit \
             --cov CodeEntropy \
             --cov-report term-missing \
             --cov-report xml \
             -q
 
-      - name: Upload to Coveralls
+      - name: Upload coverage to Coveralls
         uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/weekly-regression.yaml

@@ -2,50 +2,89 @@ name: Weekly Regression Tests
 
 on:
   schedule:
-    - cron: '0 8 * * 1'  # Weekly Monday checks
+    - cron: '0 8 * * 1'
   workflow_dispatch:
 
 concurrency:
   group: weekly-regression-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  discover:
+    name: Discover regression systems
+    runs-on: ubuntu-24.04
+
+    outputs:
+      systems: ${{ steps.set.outputs.systems }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.14"
+          cache: pip
+
+      - name: Install testing dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .[testing]
+
+      - name: Generate system matrix
+        id: set
+        run: |
+          systems=$(python -m tests.regression.list_systems)
+          echo "systems=$systems" >> $GITHUB_OUTPUT
+
   regression:
-    name: Regression tests (including slow)
+    name: Regression (${{ matrix.system }})
+    needs: discover
     runs-on: ubuntu-24.04
+    timeout-minutes: 360
+
+    strategy:
+      fail-fast: false
+      matrix:
+        system: ${{ fromJson(needs.discover.outputs.systems) }}
+
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+      - name: Checkout
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Python 3.14
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: "3.14"
           cache: pip
 
-      - name: Cache regression test data downloads
+      - name: Cache regression test data
         uses: actions/cache@v4
         with:
           path: .testdata
           key: codeentropy-testdata-${{ runner.os }}-py314
 
-      - name: Install CodeEntropy and its testing dependencies
+      - name: Install testing dependencies
         run: |
-          pip install --upgrade pip
-          pip install -e .[testing]
+          python -m pip install --upgrade pip
+          python -m pip install -e .[testing]
 
-      - name: Run regression test suite
-        run: pytest tests/regression --run-slow
+      - name: Run regression tests (slow, per system)
+        run: |
+          pytest tests/regression \
+            -k "${{ matrix.system }}" \
+            --run-slow \
+            -n auto \
+            --dist=loadscope \
+            -vv \
+            --durations=20
 
-      - name: Upload regression artifacts on failure
+      - name: Upload regression artifacts (failure)
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: regression-failure-artifacts
+          name: regression-${{ matrix.system }}-artifacts
           path: |
             .testdata/**
-            tests/regression/**/.pytest_cache/**
-            /tmp/pytest-of-*/pytest-*/**/config.yaml
-            /tmp/pytest-of-*/pytest-*/**/codeentropy_stdout.txt
-            /tmp/pytest-of-*/pytest-*/**/codeentropy_stderr.txt
-            /tmp/pytest-of-*/pytest-*/**/codeentropy_output.json
+            /tmp/pytest-of-*/pytest-*/**

.gitignore

@@ -129,4 +129,4 @@ job*
 .testdata/
 
 !tests/regression/baselines/
-!tests/regression/baselines/*.json
+!tests/regression/baselines/*/*.json

conda-recipe/meta.yaml

@@ -38,6 +38,7 @@ requirements:
     - dask >=2026.1.2,<2026.2.0
     - distributed >=2026.1.2,<2026.2.0
     - dask-jobqueue >=0.9,<0.10
+    - pytest-xdist >=3.8, <3.9
 
 test:
   imports:

docs/developer_guide.rst

@@ -54,7 +54,7 @@ Run regression tests excluding slow systems::
 
 Run slow regression tests::
 
-    pytest tests/regression -m slow
+    pytest tests/regression --run-slow
 
 Run tests with coverage::
 

pyproject.toml

@@ -61,7 +61,8 @@ Documentation = "https://codeentropy.readthedocs.io"
 testing = [
     "pytest>=9.0,<10.0",
     "pytest-cov>=7.0,<8.0",
-    "pytest-sugar>=1.1,<2.0"
+    "pytest-sugar>=1.1,<2.0",
+    "pytest-xdist"
 ]
 pre-commit = [
     "pre-commit>=4.5,<5.0",
@@ -83,6 +84,16 @@ docs = [
 [project.scripts]
 CodeEntropy = "CodeEntropy.cli:main"
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+addopts = "-n auto -vv -ra"
+
+markers = [
+  "regression: end-to-end regression tests against baselines",
+  "slow: long-running regression tests (20-30+ minutes)",
+]
+
 [tool.ruff]
 line-length = 88
 target-version = "py311"

tests/__init__.py

@@ -1,4 +1 @@
-"""
-Empty init file in case you choose a package besides PyTest such as Nose which may look
-for such a file.
-"""
+""" """