From 0fbd69e4f6b9ec3de1ddaa986e944aad6cb0afab Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sun, 12 Apr 2026 04:49:24 -0700 Subject: [PATCH 1/6] Add coverage enforcement: fail-under 78% and diff-cover 90% - Add --cov-fail-under=78 to all unit-test Makefile targets to prevent overall coverage regression (current baseline: ~79%) - Add diff-cover>=9.0.0 as dev dependency - Add diff-cover step to CI workflow (PR-only) requiring >=90% coverage on new/changed lines - Add 'make diff-cover' target for local use Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build_and_test.yml | 7 +++++++ Makefile | 12 ++++++++---- pyproject.toml | 1 + 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3f677f2767..bfebfcfd35 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -125,6 +125,13 @@ jobs: - name: Run unit tests with code coverage run: make unit-test-cov-xml + - name: Check diff coverage on PR + if: github.event_name == 'pull_request' + shell: bash + run: | + git fetch origin main + uv run -m diff_cover coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 + - name: Publish Pytest Results uses: EnricoMi/publish-unit-test-result-action@v2 if: runner.os == 'ubuntu-latest' diff --git a/Makefile b/Makefile index c3a251059a..411e4f8b91 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all pre-commit mypy test test-cov-html test-cov-xml +.PHONY: all pre-commit mypy test test-cov-html test-cov-xml diff-cover CMD:=uv run -m PYMODULE:=pyrit @@ -35,13 +35,17 @@ docs-api: # Because of import time, "auto" seemed to actually go slower than just using 4 processes unit-test: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) unit-test-cov-html: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report html + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report html unit-test-cov-xml: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules + +diff-cover: + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report xml + $(CMD) diff_cover coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 integration-test: $(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules diff --git a/pyproject.toml b/pyproject.toml index 09c477e63a..7ad7bea738 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ dev = [ "pre-commit>=4.2.0", "pytest>=8.3.5", "pytest-asyncio>=1.0.0", + "diff-cover>=9.0.0", "pytest-cov>=6.1.1", "pytest-timeout>=2.4.0", "pytest-xdist>=3.6.1", From 34ba5ccdaae4f019fa23bfa90dde30ffc8290fdb Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sun, 12 Apr 2026 04:52:17 -0700 Subject: [PATCH 2/6] Move diff-cover to separate workflow Runs in parallel with build_and_test on a single Ubuntu/Python 3.12 runner instead of blocking the full OS x Python matrix. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build_and_test.yml | 7 ----- .github/workflows/diff_cover.yml | 47 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/diff_cover.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index bfebfcfd35..3f677f2767 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -125,13 +125,6 @@ jobs: - name: Run unit tests with code coverage run: make unit-test-cov-xml - - name: Check diff coverage on PR - if: github.event_name == 'pull_request' - shell: bash - run: | - git fetch origin main - uv run -m diff_cover coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 - - name: Publish Pytest Results uses: EnricoMi/publish-unit-test-result-action@v2 if: runner.os == 'ubuntu-latest' diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml new file mode 100644 index 0000000000..66019af934 --- /dev/null +++ b/.github/workflows/diff_cover.yml @@ -0,0 +1,47 @@ +# Checks that new/changed lines in PRs have adequate test coverage + +name: diff_cover + +on: + pull_request: + branches: + - "main" + - "release/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + diff-cover: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v3 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.9.17" + enable-cache: true + cache-dependency-glob: | + **/pyproject.toml + **/uv.lock + + - name: Install dev extras + run: uv sync --extra dev --extra all + + - name: Run unit tests with coverage + run: | + uv run -m pytest -n 4 --dist=loadfile --cov=pyrit tests/unit --cov-report xml -q --no-header + + - name: Check diff coverage (>=90% on changed lines) + run: | + uv run -m diff_cover coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 From 3740b6aaf585ba49a8035c903a99d44d08e8adc6 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sun, 12 Apr 2026 05:09:57 -0700 Subject: [PATCH 3/6] Fix diff-cover invocation and flaky setup test - Use 'python -m diff_cover.diff_cover_tool' instead of '-m diff_cover' (diff_cover is a package, not directly executable) - Fix test_simple_initializer assertion: check for 'converter_target' instead of 'PromptConverter.converter_target' since apply_defaults registers concrete class names, not the abstract base Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/diff_cover.yml | 2 +- Makefile | 2 +- tests/unit/setup/test_simple_initializer.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml index 66019af934..a6dbd471f8 100644 --- a/.github/workflows/diff_cover.yml +++ b/.github/workflows/diff_cover.yml @@ -44,4 +44,4 @@ jobs: - name: Check diff coverage (>=90% on changed lines) run: | - uv run -m diff_cover coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 + uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 diff --git a/Makefile b/Makefile index 411e4f8b91..432b162df2 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ unit-test-cov-xml: diff-cover: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report xml - $(CMD) diff_cover coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 + uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 integration-test: $(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules diff --git a/tests/unit/setup/test_simple_initializer.py b/tests/unit/setup/test_simple_initializer.py index a7960fc6cd..72cd526074 100644 --- a/tests/unit/setup/test_simple_initializer.py +++ b/tests/unit/setup/test_simple_initializer.py @@ -92,8 +92,8 @@ async def test_get_info_after_initialize_has_populated_data(self): # Verify expected default values are present default_values_str = str(info["default_values"]) - assert "PromptConverter.converter_target" in default_values_str - assert "PromptSendingAttack.attack_scoring_config" in default_values_str + assert "converter_target" in default_values_str + assert "attack_scoring_config" in default_values_str # Verify global_variables list is populated and not empty assert isinstance(info["global_variables"], list) From e62be33fc3936e5a113bf087c863989876ef16a5 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sun, 12 Apr 2026 05:14:32 -0700 Subject: [PATCH 4/6] Consolidate coverage into single workflow - Remove --cov from build_and_test.yml matrix jobs (24 jobs now run faster without coverage overhead) - Move CodeCoverageSummary report to coverage workflow - coverage workflow runs once on Ubuntu/Python 3.12: 1. Overall threshold: --cov-fail-under=78 2. Coverage summary report (badge + markdown) 3. Diff coverage on PRs: >=90% on changed lines - Rename diff_cover.yml workflow to 'coverage' - Add unit-test-cov Makefile target (local coverage without xml/junit) - Keep unit-test target clean (no coverage, for quick local runs) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build_and_test.yml | 19 +++-------------- .github/workflows/diff_cover.yml | 32 ++++++++++++++++++++++------ Makefile | 5 ++++- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3f677f2767..acc9dc0356 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -122,25 +122,12 @@ jobs: uv sync --extra dev fi - - name: Run unit tests with code coverage - run: make unit-test-cov-xml + - name: Run unit tests + run: | + uv run -m pytest -n 4 --dist=loadfile tests/unit --junitxml=junit/test-results.xml - name: Publish Pytest Results uses: EnricoMi/publish-unit-test-result-action@v2 if: runner.os == 'ubuntu-latest' with: files: '**/test-*.xml' - - - name: Code Coverage Report - uses: irongut/CodeCoverageSummary@v1.3.0 - if: runner.os == 'ubuntu-latest' - with: - filename: coverage.xml - badge: true - fail_below_min: false - format: markdown - hide_branch_rate: false - hide_complexity: true - indicators: true - output: both - thresholds: '60 80' diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml index a6dbd471f8..ecff391d50 100644 --- a/.github/workflows/diff_cover.yml +++ b/.github/workflows/diff_cover.yml @@ -1,8 +1,12 @@ -# Checks that new/changed lines in PRs have adequate test coverage +# Single place for all coverage checks: overall threshold + diff coverage on PRs. +# Runs once on Ubuntu/Python 3.12 instead of across the full OS x Python matrix. -name: diff_cover +name: coverage on: + push: + branches: + - "main" pull_request: branches: - "main" @@ -13,7 +17,7 @@ concurrency: cancel-in-progress: true jobs: - diff-cover: + coverage: runs-on: ubuntu-latest permissions: contents: read @@ -38,10 +42,26 @@ jobs: - name: Install dev extras run: uv sync --extra dev --extra all - - name: Run unit tests with coverage + - name: Run unit tests with coverage (fail under 78%) run: | - uv run -m pytest -n 4 --dist=loadfile --cov=pyrit tests/unit --cov-report xml -q --no-header + uv run -m pytest -n 4 --dist=loadfile --cov=pyrit --cov-fail-under=78 tests/unit --cov-report xml --cov-report term + + - name: Code Coverage Report + uses: irongut/CodeCoverageSummary@v1.3.0 + if: always() + with: + filename: coverage.xml + badge: true + fail_below_min: false + format: markdown + hide_branch_rate: false + hide_complexity: true + indicators: true + output: both + thresholds: '60 80' - - name: Check diff coverage (>=90% on changed lines) + - name: Check diff coverage on PR (>=90% on changed lines) + if: github.event_name == 'pull_request' run: | + git fetch origin main uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 diff --git a/Makefile b/Makefile index 432b162df2..591ce38935 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,9 @@ docs-api: # Because of import time, "auto" seemed to actually go slower than just using 4 processes unit-test: + $(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS) + +unit-test-cov: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) unit-test-cov-html: @@ -44,7 +47,7 @@ unit-test-cov-xml: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules diff-cover: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report xml + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 integration-test: From 9b624e8ff1eb08f0a1f5dd4aac0abc34535a37ec Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sun, 12 Apr 2026 05:26:38 -0700 Subject: [PATCH 5/6] Use Makefile targets in all CI workflows - Add unit-test-junit target (tests + junit xml, no coverage) - Add diff-cover-only target (just diff-cover, assumes coverage.xml exists) - Both workflows now use make instead of inline commands - Removed --doctest-modules from unit-test-cov-xml (not needed for coverage) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build_and_test.yml | 3 +-- .github/workflows/diff_cover.yml | 5 ++--- Makefile | 10 ++++++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index acc9dc0356..b1d850e68c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -123,8 +123,7 @@ jobs: fi - name: Run unit tests - run: | - uv run -m pytest -n 4 --dist=loadfile tests/unit --junitxml=junit/test-results.xml + run: make unit-test-junit - name: Publish Pytest Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml index ecff391d50..9ece9a7dd6 100644 --- a/.github/workflows/diff_cover.yml +++ b/.github/workflows/diff_cover.yml @@ -43,8 +43,7 @@ jobs: run: uv sync --extra dev --extra all - name: Run unit tests with coverage (fail under 78%) - run: | - uv run -m pytest -n 4 --dist=loadfile --cov=pyrit --cov-fail-under=78 tests/unit --cov-report xml --cov-report term + run: make unit-test-cov-xml - name: Code Coverage Report uses: irongut/CodeCoverageSummary@v1.3.0 @@ -64,4 +63,4 @@ jobs: if: github.event_name == 'pull_request' run: | git fetch origin main - uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 + make diff-cover-only diff --git a/Makefile b/Makefile index 591ce38935..e41bd88b9f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all pre-commit mypy test test-cov-html test-cov-xml diff-cover +.PHONY: all pre-commit mypy unit-test unit-test-junit unit-test-cov unit-test-cov-html unit-test-cov-xml diff-cover CMD:=uv run -m PYMODULE:=pyrit @@ -37,6 +37,9 @@ docs-api: unit-test: $(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS) +unit-test-junit: + $(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS) --junitxml=junit/test-results.xml + unit-test-cov: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) @@ -44,12 +47,15 @@ unit-test-cov-html: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report html unit-test-cov-xml: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml --cov-report term diff-cover: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 +diff-cover-only: + uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 + integration-test: $(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules From 643f8c96710119958dbb64c6f85321e97563e92b Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Mon, 13 Apr 2026 10:31:46 -0700 Subject: [PATCH 6/6] Address review: rename diff-cover-only, remove unit-test-cov - Rename diff-cover-only -> unit-test-diff-cover (consistent naming) - Remove unit-test-cov target (use cov-html or cov-xml instead) - Update workflow to use renamed target Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/diff_cover.yml | 2 +- Makefile | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml index 9ece9a7dd6..12caaf31e2 100644 --- a/.github/workflows/diff_cover.yml +++ b/.github/workflows/diff_cover.yml @@ -63,4 +63,4 @@ jobs: if: github.event_name == 'pull_request' run: | git fetch origin main - make diff-cover-only + make unit-test-diff-cover diff --git a/Makefile b/Makefile index e41bd88b9f..8a0c1a4c93 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all pre-commit mypy unit-test unit-test-junit unit-test-cov unit-test-cov-html unit-test-cov-xml diff-cover +.PHONY: all pre-commit mypy unit-test unit-test-junit unit-test-cov-html unit-test-cov-xml diff-cover unit-test-diff-cover CMD:=uv run -m PYMODULE:=pyrit @@ -40,9 +40,6 @@ unit-test: unit-test-junit: $(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS) --junitxml=junit/test-results.xml -unit-test-cov: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) - unit-test-cov-html: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report html @@ -53,7 +50,7 @@ diff-cover: $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 -diff-cover-only: +unit-test-diff-cover: uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 integration-test: