diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3f677f2767..b1d850e68c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -122,25 +122,11 @@ jobs: uv sync --extra dev fi - - name: Run unit tests with code coverage - run: make unit-test-cov-xml + - name: Run unit tests + run: make unit-test-junit - name: Publish Pytest Results uses: EnricoMi/publish-unit-test-result-action@v2 if: runner.os == 'ubuntu-latest' with: files: '**/test-*.xml' - - - name: Code Coverage Report - uses: irongut/CodeCoverageSummary@v1.3.0 - if: runner.os == 'ubuntu-latest' - with: - filename: coverage.xml - badge: true - fail_below_min: false - format: markdown - hide_branch_rate: false - hide_complexity: true - indicators: true - output: both - thresholds: '60 80' diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml new file mode 100644 index 0000000000..12caaf31e2 --- /dev/null +++ b/.github/workflows/diff_cover.yml @@ -0,0 +1,66 @@ +# Single place for all coverage checks: overall threshold + diff coverage on PRs. +# Runs once on Ubuntu/Python 3.12 instead of across the full OS x Python matrix. + +name: coverage + +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + - "release/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + coverage: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v3 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.9.17" + enable-cache: true + cache-dependency-glob: | + **/pyproject.toml + **/uv.lock + + - name: Install dev extras + run: uv sync --extra dev --extra all + + - name: Run unit tests with coverage (fail under 78%) + run: make unit-test-cov-xml + + - name: Code Coverage Report + uses: irongut/CodeCoverageSummary@v1.3.0 + if: always() + with: + filename: coverage.xml + badge: true + fail_below_min: false + format: markdown + hide_branch_rate: false + hide_complexity: true + indicators: true + output: both + thresholds: '60 80' + + - name: Check diff coverage on PR (>=90% on changed lines) + if: github.event_name == 'pull_request' + run: | + git fetch origin main + make unit-test-diff-cover diff --git a/Makefile b/Makefile index c3a251059a..8a0c1a4c93 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all pre-commit mypy test test-cov-html test-cov-xml +.PHONY: all pre-commit mypy unit-test unit-test-junit unit-test-cov-html unit-test-cov-xml diff-cover unit-test-diff-cover CMD:=uv run -m PYMODULE:=pyrit @@ -35,13 +35,23 @@ docs-api: # Because of import time, "auto" seemed to actually go slower than just using 4 processes unit-test: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) + $(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS) + +unit-test-junit: + $(CMD) pytest -n 4 --dist=loadfile $(UNIT_TESTS) --junitxml=junit/test-results.xml unit-test-cov-html: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report html + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report html unit-test-cov-xml: - $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) $(UNIT_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml --cov-report term + +diff-cover: + $(CMD) pytest -n 4 --dist=loadfile --cov=$(PYMODULE) --cov-fail-under=78 $(UNIT_TESTS) --cov-report xml + uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 + +unit-test-diff-cover: + uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 integration-test: $(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules diff --git a/pyproject.toml b/pyproject.toml index 2a349f9f53..83aeba04f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ dev = [ "pre-commit>=4.2.0", "pytest>=8.3.5", "pytest-asyncio>=1.0.0", + "diff-cover>=9.0.0", "pytest-cov>=6.1.1", "pytest-timeout>=2.4.0", "pytest-xdist>=3.6.1", diff --git a/tests/unit/setup/test_simple_initializer.py b/tests/unit/setup/test_simple_initializer.py index a7960fc6cd..72cd526074 100644 --- a/tests/unit/setup/test_simple_initializer.py +++ b/tests/unit/setup/test_simple_initializer.py @@ -92,8 +92,8 @@ async def test_get_info_after_initialize_has_populated_data(self): # Verify expected default values are present default_values_str = str(info["default_values"]) - assert "PromptConverter.converter_target" in default_values_str - assert "PromptSendingAttack.attack_scoring_config" in default_values_str + assert "converter_target" in default_values_str + assert "attack_scoring_config" in default_values_str # Verify global_variables list is populated and not empty assert isinstance(info["global_variables"], list)