TEST: Add coverage enforcement (fail-under 78%, diff-cover 90%)

.github/workflows/build_and_test.yml

@@ -122,25 +122,11 @@ jobs:
             uv sync --extra dev
           fi
 
-      - name: Run unit tests with code coverage
-        run: make unit-test-cov-xml
+      - name: Run unit tests
+        run: make unit-test-junit
 
       - name: Publish Pytest Results
         uses: EnricoMi/publish-unit-test-result-action@v2
         if: runner.os == 'ubuntu-latest'
         with:
           files: '**/test-*.xml'
-
-      - name: Code Coverage Report
-        uses: irongut/CodeCoverageSummary@v1.3.0
-        if: runner.os == 'ubuntu-latest'
-        with:
-          filename: coverage.xml
-          badge: true
-          fail_below_min: false
-          format: markdown
-          hide_branch_rate: false
-          hide_complexity: true
-          indicators: true
-          output: both
-          thresholds: '60 80'

.github/workflows/diff_cover.yml

@@ -0,0 +1,66 @@
+# Single place for all coverage checks: overall threshold + diff coverage on PRs.
+# Runs once on Ubuntu/Python 3.12 instead of across the full OS x Python matrix.
+
+name: coverage
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+    branches:
+      - "main"
+      - "release/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.9.17"
+          enable-cache: true
+          cache-dependency-glob: |
+            **/pyproject.toml
+            **/uv.lock
+
+      - name: Install dev extras
+        run: uv sync --extra dev --extra all
+
+      - name: Run unit tests with coverage (fail under 78%)
+        run: make unit-test-cov-xml
+
+      - name: Code Coverage Report
+        uses: irongut/CodeCoverageSummary@v1.3.0
+        if: always()
+        with:
+          filename: coverage.xml
+          badge: true
+          fail_below_min: false
+          format: markdown
+          hide_branch_rate: false
+          hide_complexity: true
+          indicators: true
+          output: both
+          thresholds: '60 80'
+
+      - name: Check diff coverage on PR (>=90% on changed lines)
+        if: github.event_name == 'pull_request'
+        run: |
+          git fetch origin main
+          make diff-cover-only

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all pre-commit mypy test test-cov-html test-cov-xml
+.PHONY: all pre-commit mypy unit-test unit-test-junit unit-test-cov unit-test-cov-html unit-test-cov-xml diff-cover
 
 CMD:=uv run -m
 PYMODULE:=pyrit
@@ -35,13 +35,26 @@ docs-api:
 
 # Because of import time, "auto" seemed to actually go slower than just using 4 processes
 unit-test:
-	$(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS)
+	$(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS)
+
+unit-test-junit:
+	$(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS) --junitxml=junit/test-results.xml
+
+unit-test-cov:
+	$(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS)
 
 unit-test-cov-html:
-	$(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report html
+	$(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report html
 
 unit-test-cov-xml:
-	$(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules
+	$(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml --cov-report term
+
+diff-cover:
+	$(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml
+	uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90
+
+diff-cover-only:
+	uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90
 
 integration-test:
 	$(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules

pyproject.toml

@@ -82,6 +82,7 @@ dev = [
     "pre-commit>=4.2.0",
     "pytest>=8.3.5",
     "pytest-asyncio>=1.0.0",
+    "diff-cover>=9.0.0",
     "pytest-cov>=6.1.1",
     "pytest-timeout>=2.4.0",
     "pytest-xdist>=3.6.1",

tests/unit/setup/test_simple_initializer.py

@@ -92,8 +92,8 @@ async def test_get_info_after_initialize_has_populated_data(self):
 
         # Verify expected default values are present
         default_values_str = str(info["default_values"])
-        assert "PromptConverter.converter_target" in default_values_str
-        assert "PromptSendingAttack.attack_scoring_config" in default_values_str
+        assert "converter_target" in default_values_str
+        assert "attack_scoring_config" in default_values_str
 
         # Verify global_variables list is populated and not empty
         assert isinstance(info["global_variables"], list)