diff --git a/.github/scripts/check_version_uniqueness.py b/.github/scripts/check_version_uniqueness.py new file mode 100644 index 000000000..48697a5f2 --- /dev/null +++ b/.github/scripts/check_version_uniqueness.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Ensure every changed package has a version not yet published to PyPI. + +For each package with file changes in the PR, we read its version from +pyproject.toml and check whether that exact version exists on PyPI. +If it does, the check fails — the developer must bump the version. + +This catches two scenarios: + 1. Developer changed code but forgot to bump the version. + 2. Two PRs raced to the same version — after rebase the first PR's + version is already on PyPI, so this check forces the second PR + to pick a new one. +""" + +import os +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore[no-redef] + + +def version_exists_on_pypi(package_name: str, version: str) -> bool | None: + url = f"https://pypi.org/pypi/{package_name}/{version}/json" + try: + req = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(req, timeout=10): + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + print(f"Warning: PyPI returned HTTP {e.code} for {package_name}=={version}", file=sys.stderr) + return None + except Exception as e: + print(f"Warning: Could not reach PyPI for {package_name}=={version}: {e}", file=sys.stderr) + return None + + +def get_package_info(package_dir: str) -> tuple[str, str] | None: + pyproject = Path("packages") / package_dir / "pyproject.toml" + if not pyproject.exists(): + return None + with open(pyproject, "rb") as f: + data = tomllib.load(f) + project = data.get("project", {}) + name = project.get("name") + version = project.get("version") + if name and version: + return name, version + return None + + +def get_changed_packages() -> list[str]: + base_sha = os.getenv("BASE_SHA", "") + head_sha = os.getenv("HEAD_SHA", "") + diff_spec = f"{base_sha}...{head_sha}" if base_sha and head_sha else "origin/main...HEAD" + + try: + result = subprocess.run( + ["git", "diff", "--name-only", diff_spec], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"Error running git diff: {e}", file=sys.stderr) + return [] + + changed = set() + for file_path in result.stdout.strip().split("\n"): + if file_path.startswith("packages/"): + parts = file_path.split("/") + if len(parts) >= 2 and (Path("packages") / parts[1] / "pyproject.toml").exists(): + changed.add(parts[1]) + return sorted(changed) + + +def main() -> int: + changed = get_changed_packages() + if not changed: + print("No changed packages detected.") + return 0 + + conflicts = [] + for pkg_dir in changed: + info = get_package_info(pkg_dir) + if not info: + continue + + name, version = info + exists = version_exists_on_pypi(name, version) + + if exists is True: + print(f"FAIL: {name}=={version} already exists on PyPI") + conflicts.append(f"{name}=={version}") + elif exists is False: + print(f"OK: {name}=={version} is available") + else: + print(f"SKIP: could not verify {name}=={version}") + + if conflicts: + print(f"\nPlease bump the version in pyproject.toml for: {', '.join(conflicts)}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_check_version_uniqueness.py b/.github/scripts/test_check_version_uniqueness.py new file mode 100644 index 000000000..67a586668 --- /dev/null +++ b/.github/scripts/test_check_version_uniqueness.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Tests for check_version_uniqueness.py.""" + +import os +import urllib.error +from unittest import mock + +import pytest + +from check_version_uniqueness import ( + get_package_info, + main, + version_exists_on_pypi, +) + + +class TestVersionExistsOnPypi: + def test_returns_true_when_version_exists(self): + mock_response = mock.MagicMock() + mock_response.__enter__ = mock.MagicMock(return_value=mock_response) + mock_response.__exit__ = mock.MagicMock(return_value=False) + + with mock.patch("urllib.request.urlopen", return_value=mock_response): + assert version_exists_on_pypi("some-package", "1.0.0") is True + + def test_returns_false_when_404(self): + error = urllib.error.HTTPError( + url="", code=404, msg="Not Found", hdrs=None, fp=None # type: ignore[arg-type] + ) + with mock.patch("urllib.request.urlopen", side_effect=error): + assert version_exists_on_pypi("some-package", "9.9.9") is False + + def test_returns_none_on_server_error(self): + error = urllib.error.HTTPError( + url="", code=500, msg="Server Error", hdrs=None, fp=None # type: ignore[arg-type] + ) + with mock.patch("urllib.request.urlopen", side_effect=error): + assert version_exists_on_pypi("some-package", "1.0.0") is None + + def test_returns_none_on_network_error(self): + with mock.patch( + "urllib.request.urlopen", side_effect=ConnectionError("no network") + ): + assert version_exists_on_pypi("some-package", "1.0.0") is None + + +class TestGetPackageInfo: + def test_reads_name_and_version(self, tmp_path): + pkg = tmp_path / "packages" / "my-pkg" + pkg.mkdir(parents=True) + (pkg / "pyproject.toml").write_text( + '[project]\nname = "my-pkg"\nversion = "1.2.3"\n' + ) + with mock.patch( + "check_version_uniqueness.Path", + side_effect=lambda p: tmp_path / p if p == "packages" else __import__("pathlib").Path(p), + ): + assert get_package_info("my-pkg") == ("my-pkg", "1.2.3") + + def test_returns_none_for_missing_file(self): + assert get_package_info("nonexistent-package-xyz") is None + + +class TestMain: + def _run(self, changed, package_info, pypi_result): + patches = [ + mock.patch("check_version_uniqueness.get_changed_packages", return_value=changed), + mock.patch("check_version_uniqueness.get_package_info", side_effect=lambda d: package_info.get(d)), + mock.patch("check_version_uniqueness.version_exists_on_pypi", return_value=pypi_result), + ] + with patches[0], patches[1], patches[2]: + os.environ.pop("BASE_SHA", None) + os.environ.pop("HEAD_SHA", None) + return main() + + def test_passes_when_version_not_on_pypi(self): + assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "2.0.0")}, False) == 0 + + def test_fails_when_version_exists_on_pypi(self): + assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "2.0.0")}, True) == 1 + + def test_passes_on_network_error(self): + assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "2.0.0")}, None) == 0 + + def test_no_changed_packages(self): + assert self._run([], {}, False) == 0 + + def test_fails_when_version_unchanged_but_on_pypi(self): + """The key scenario: code changed, version not bumped, already on PyPI.""" + assert self._run(["my-pkg"], {"my-pkg": ("my-pkg", "1.0.0")}, True) == 1 + + def test_multiple_packages_one_conflict(self): + def pypi_check(name, version): + return name == "pkg-a" + + with ( + mock.patch("check_version_uniqueness.get_changed_packages", return_value=["pkg-a", "pkg-b"]), + mock.patch( + "check_version_uniqueness.get_package_info", + side_effect=lambda d: {"pkg-a": ("pkg-a", "1.0.0"), "pkg-b": ("pkg-b", "2.0.0")}.get(d), + ), + mock.patch("check_version_uniqueness.version_exists_on_pypi", side_effect=pypi_check), + ): + os.environ.pop("BASE_SHA", None) + os.environ.pop("HEAD_SHA", None) + assert main() == 1 diff --git a/.github/scripts/wait_for_pypi.py b/.github/scripts/wait_for_pypi.py new file mode 100644 index 000000000..bc2186e03 --- /dev/null +++ b/.github/scripts/wait_for_pypi.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Wait for a package version to become available on PyPI. + +Usage: python wait_for_pypi.py + +Reads the package name and version from packages//pyproject.toml, +then polls PyPI until the version appears or a timeout is reached. +""" + +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore[no-redef] + +MAX_WAIT = 300 # 5 minutes +POLL_INTERVAL = 15 # seconds + + +def version_exists_on_pypi(package_name: str, version: str) -> bool: + """Check if a specific version of a package exists on PyPI.""" + url = f"https://pypi.org/pypi/{package_name}/{version}/json" + try: + req = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(req, timeout=10): + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + print(f" Warning: PyPI returned HTTP {e.code}", file=sys.stderr) + return False + except Exception as e: + print(f" Warning: Could not reach PyPI: {e}", file=sys.stderr) + return False + + +def main() -> int: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + return 1 + + directory = sys.argv[1] + pyproject = Path("packages") / directory / "pyproject.toml" + + if not pyproject.exists(): + print(f"ERROR: {pyproject} not found", file=sys.stderr) + return 1 + + with open(pyproject, "rb") as f: + data = tomllib.load(f) + + name = data["project"]["name"] + version = data["project"]["version"] + + print(f"Waiting for {name}=={version} to appear on PyPI...") + elapsed = 0 + while elapsed < MAX_WAIT: + if version_exists_on_pypi(name, version): + print(f"{name}=={version} is now available on PyPI.") + return 0 + time.sleep(POLL_INTERVAL) + elapsed += POLL_INTERVAL + print(f" Still waiting... ({elapsed}s/{MAX_WAIT}s)") + + print( + f"ERROR: {name}=={version} did not appear on PyPI within {MAX_WAIT}s", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 04e9e302a..7c3a99dd9 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -33,83 +33,88 @@ jobs: id: detect run: python .github/scripts/detect_publishable_packages.py - build: - name: Build ${{ matrix.package }} + # --- Tier 0: uipath-core (no internal dependencies) --- + publish-uipath-core: needs: detect-publishable-packages - if: needs.detect-publishable-packages.outputs.count > 0 && github.repository == 'UiPath/uipath-python' + if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core') + uses: ./.github/workflows/publish-package.yml + with: + package: uipath-core + needs-relock: false + + wait-for-uipath-core: + name: Wait for uipath-core on PyPI + needs: [detect-publishable-packages, publish-uipath-core] + if: | + always() && + (contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') || + contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath')) runs-on: ubuntu-latest - defaults: - run: - working-directory: packages/${{ matrix.package }} - strategy: - fail-fast: false - matrix: - package: ${{ fromJson(needs.detect-publishable-packages.outputs.packages) }} - permissions: - contents: read - actions: write - steps: - name: Checkout + if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core') uses: actions/checkout@v4 - - name: Setup uv - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - name: Setup Python + if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core') uses: actions/setup-python@v5 with: - python-version-file: "packages/${{ matrix.package }}/.python-version" - - - name: Install dependencies - run: uv sync --all-extras - - - name: Update AGENTS.md - if: matrix.package == 'uipath' - run: uv run python scripts/update_agents_md.py - - - name: Replace connection string placeholder - if: matrix.package == 'uipath' - run: | - originalfile="src/uipath/telemetry/_constants.py" - tmpfile=$(mktemp) - trap 'rm -f "$tmpfile"' EXIT - - rsync -a --no-whole-file --ignore-existing "$originalfile" "$tmpfile" - envsubst '$CONNECTION_STRING' < "$originalfile" > "$tmpfile" && mv "$tmpfile" "$originalfile" - env: - CONNECTION_STRING: ${{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING }} - - - name: Build - run: uv build --no-sources --package ${{ matrix.package }} - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: release-dists-${{ matrix.package }} - path: packages/${{ matrix.package }}/dist/ + python-version: '3.11' - pypi-publish: - name: Upload ${{ matrix.package }} to PyPI - needs: [detect-publishable-packages, build] + - name: Wait for uipath-core + if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core') + run: python .github/scripts/wait_for_pypi.py uipath-core + + - name: Skip + if: "!contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core')" + run: echo "uipath-core not being published — skipping wait" + + # --- Tier 1: uipath-platform (depends on core) --- + publish-uipath-platform: + needs: [detect-publishable-packages, wait-for-uipath-core] + if: | + always() && + contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') + uses: ./.github/workflows/publish-package.yml + with: + package: uipath-platform + needs-relock: true + + wait-for-uipath-platform: + name: Wait for uipath-platform on PyPI + needs: [detect-publishable-packages, publish-uipath-platform] + if: | + always() && + contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath') runs-on: ubuntu-latest - environment: pypi - strategy: - fail-fast: false - matrix: - package: ${{ fromJson(needs.detect-publishable-packages.outputs.packages) }} - permissions: - contents: read - id-token: write - steps: - - name: Retrieve release distributions - uses: actions/download-artifact@v4 + - name: Checkout + if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') + uses: actions/checkout@v4 + + - name: Setup Python + if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') + uses: actions/setup-python@v5 with: - name: release-dists-${{ matrix.package }} - path: dist/ + python-version: '3.11' - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Wait for uipath-platform + if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') + run: python .github/scripts/wait_for_pypi.py uipath-platform + + - name: Skip + if: "!contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform')" + run: echo "uipath-platform not being published — skipping wait" + + # --- Tier 2: uipath (depends on core + platform) --- + publish-uipath: + needs: [detect-publishable-packages, wait-for-uipath-platform] + if: | + always() && + contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath') + uses: ./.github/workflows/publish-package.yml + with: + package: uipath + needs-relock: true + secrets: + APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING }} diff --git a/.github/workflows/check-version-availability.yml b/.github/workflows/check-version-availability.yml new file mode 100644 index 000000000..02ace93c1 --- /dev/null +++ b/.github/workflows/check-version-availability.yml @@ -0,0 +1,28 @@ +name: Check Version Availability + +on: + workflow_call: + +permissions: + contents: read + +jobs: + check-version-availability: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check changed package versions are not on PyPI + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python .github/scripts/check_version_uniqueness.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18eb9f724..5e05e9c73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,3 +18,6 @@ jobs: test: uses: ./.github/workflows/test-packages.yml + + check-versions: + uses: ./.github/workflows/check-version-availability.yml diff --git a/.github/workflows/lint-packages.yml b/.github/workflows/lint-packages.yml index e7f275c54..d530e25a9 100644 --- a/.github/workflows/lint-packages.yml +++ b/.github/workflows/lint-packages.yml @@ -68,7 +68,7 @@ jobs: - name: Install dependencies if: steps.check.outputs.skip != 'true' working-directory: packages/uipath-core - run: uv sync --locked --no-sources --all-extras + run: uv sync --locked --all-extras - name: Check static types if: steps.check.outputs.skip != 'true' @@ -122,7 +122,7 @@ jobs: - name: Install dependencies if: steps.check.outputs.skip != 'true' working-directory: packages/uipath-platform - run: uv sync --locked --no-sources --all-extras + run: uv sync --locked --all-extras - name: Check static types if: steps.check.outputs.skip != 'true' @@ -176,7 +176,7 @@ jobs: - name: Install dependencies if: steps.check.outputs.skip != 'true' working-directory: packages/uipath - run: uv sync --locked --no-sources --all-extras + run: uv sync --locked --all-extras - name: Check static types if: steps.check.outputs.skip != 'true' diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml new file mode 100644 index 000000000..5951a7acb --- /dev/null +++ b/.github/workflows/publish-package.yml @@ -0,0 +1,92 @@ +name: Publish Package + +on: + workflow_call: + inputs: + package: + description: 'Package directory name (e.g. uipath-core)' + required: true + type: string + needs-relock: + description: 'Whether to re-lock against PyPI before building (for packages with internal dependencies)' + required: false + type: boolean + default: false + secrets: + APPLICATIONINSIGHTS_CONNECTION_STRING: + required: false + +permissions: + contents: read + id-token: write + actions: write + +jobs: + build: + name: Build ${{ inputs.package }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: packages/${{ inputs.package }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version-file: "packages/${{ inputs.package }}/.python-version" + + - name: Install dependencies + run: uv sync --all-extras + + - name: Update AGENTS.md + if: inputs.package == 'uipath' + run: uv run python scripts/update_agents_md.py + + - name: Replace connection string placeholder + if: inputs.package == 'uipath' + run: | + originalfile="src/uipath/telemetry/_constants.py" + tmpfile=$(mktemp) + trap 'rm -f "$tmpfile"' EXIT + rsync -a --no-whole-file --ignore-existing "$originalfile" "$tmpfile" + envsubst '$CONNECTION_STRING' < "$originalfile" > "$tmpfile" && mv "$tmpfile" "$originalfile" + env: + CONNECTION_STRING: ${{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING }} + + - name: Re-lock against PyPI + if: inputs.needs-relock + run: uv lock --no-sources --no-cache + + - name: Build + run: uv build --no-sources --package ${{ inputs.package }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: release-dists-${{ inputs.package }} + path: packages/${{ inputs.package }}/dist/ + + publish: + name: Publish ${{ inputs.package }} + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write + steps: + - name: Retrieve release distributions + uses: actions/download-artifact@v4 + with: + name: release-dists-${{ inputs.package }} + path: dist/ + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test-cd-scripts.yml b/.github/workflows/test-cd-scripts.yml index 3dd2047e7..a2f1bcdf1 100644 --- a/.github/workflows/test-cd-scripts.yml +++ b/.github/workflows/test-cd-scripts.yml @@ -7,7 +7,10 @@ on: paths: - '.github/scripts/detect_publishable_packages.py' - '.github/scripts/test_detect_publishable_packages.py' + - '.github/scripts/check_version_uniqueness.py' + - '.github/scripts/test_check_version_uniqueness.py' - '.github/workflows/cd.yml' + - '.github/workflows/check-version-availability.yml' permissions: contents: read @@ -29,4 +32,4 @@ jobs: - name: Run tests working-directory: .github/scripts - run: python -m pytest test_detect_publishable_packages.py -v + run: python -m pytest test_detect_publishable_packages.py test_check_version_uniqueness.py -v