Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
e01c8df
test: improve test coverage for distributed and dataset crates
zhexuany Feb 23, 2026
1053444
test: add comprehensive tests for low-coverage modules
zhexuany Feb 23, 2026
8337d80
test: add comprehensive S3 storage integration tests
zhexuany Feb 23, 2026
f6c8539
docs: update test infrastructure requirements and remove ignored flag
zhexuany Feb 23, 2026
0eacfa2
fix: use canonical batch_id from spec in e2e tests
zhexuany Feb 23, 2026
8400f43
fix: use canonical batch_id in dataset_integrity_e2e tests
zhexuany Feb 23, 2026
ecf3433
test: add comprehensive tests for low-coverage modules
zhexuany Feb 23, 2026
dfde7e4
fix: add TiKV config fetching and fix bincode serialization
zhexuany Feb 23, 2026
d591f7d
refactor: extract pipeline crate and simplify executor architecture
zhexuany Feb 24, 2026
67f1827
refactor: migrate callers to pipeline executor surface
zhexuany Feb 24, 2026
60ec0a5
feat: introduce WorkProcessor abstraction for coordinator
zhexuany Feb 24, 2026
2f952ff
refactor: enforce distributed control-plane boundaries
zhexuany Feb 24, 2026
43106f6
test: add architecture-focused pipeline and batch e2e coverage
zhexuany Feb 24, 2026
2fcf53c
feat: add shared video profiles and dataset encode helpers
zhexuany Feb 24, 2026
9800f98
fix: provision MinIO and PD alias for linux e2e CI
zhexuany Feb 24, 2026
db5d21e
refactor: remove lerobot flat config compatibility parsing
zhexuany Feb 24, 2026
3a521f8
test: raise lerobot coverage and coordinator metadata behavior
zhexuany Feb 24, 2026
3e82df9
fix: harden linux CI infra readiness for TiKV tests
zhexuany Feb 24, 2026
912e415
ci: add reusable workflow actions for CI infra checks
zhexuany Feb 24, 2026
3ac5f1b
fix: correct test failures on Linux CI
zhexuany Feb 24, 2026
b581daa
fix: disable sccache wrapper in security audit job
zhexuany Feb 24, 2026
e8df46c
fix: add missing return in AVX2 load_rgb_values
zhexuany Feb 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .github/actions/cache-docker/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: 'Setup Docker'
description: 'Setup Docker Buildx and pre-pull compose images'

runs:
using: 'composite'
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Pull Docker images
shell: bash
run: docker compose pull --quiet
10 changes: 10 additions & 0 deletions .github/actions/docker-cleanup/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
name: 'Docker Cleanup'
description: 'Cleanup Docker Compose services and volumes'

runs:
using: 'composite'
steps:
- name: Docker compose down
if: always()
shell: bash
run: docker compose down -v
74 changes: 74 additions & 0 deletions .github/actions/setup-rust-env/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
name: 'Setup Rust Environment'
description: 'Setup Rust toolchain, Python, caching, and common build dependencies'

inputs:
python-version:
description: 'Python version to install'
required: false
default: '3.11'
rust-components:
description: 'Rust components to install (comma-separated)'
required: false
default: ''
cache-pip:
description: 'Enable pip caching'
required: false
default: 'false'
install-maturin:
description: 'Install maturin for building Python wheels'
required: false
default: 'false'
install-build-deps:
description: 'Install protobuf-compiler, mold, and clang on Linux'
required: false
default: 'true'

runs:
using: 'composite'
steps:
- uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
cache: ${{ inputs.cache-pip == 'true' && 'pip' || '' }}

- uses: dtolnay/rust-toolchain@stable
with:
components: ${{ inputs.rust-components }}

- name: Install Linux build dependencies
if: runner.os == 'Linux' && inputs.install-build-deps == 'true'
shell: bash
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler mold clang

- name: Detect cache key suffix
id: cache-suffix
shell: bash
run: |
if [ "${{ runner.os }}" = "Linux" ]; then
UBUNTU_VERSION=$(lsb_release -rs | tr -d '.')
echo "suffix=ubuntu-${UBUNTU_VERSION}" >> $GITHUB_OUTPUT
else
echo "suffix=${{ runner.os }}" >> $GITHUB_OUTPUT
fi

- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.9

- name: Setup cargo retry wrapper
shell: bash
run: |
mkdir -p "$HOME/.cargo-bin"
cp "${{ github.action_path }}/scripts/cargo-with-retry.sh" "$HOME/.cargo-bin/cargo-retry"
chmod +x "$HOME/.cargo-bin/cargo-retry"
echo "$HOME/.cargo-bin" >> $GITHUB_PATH
echo "SCCACHE_IGNORE_SERVER_IO_ERROR=1" >> $GITHUB_ENV

- uses: Swatinem/rust-cache@v2
with:
shared-key: rust-stable-${{ runner.os }}-${{ steps.cache-suffix.outputs.suffix }}
cache-on-failure: true

- name: Install maturin
if: inputs.install-maturin == 'true'
shell: bash
run: pip install maturin[patchelf] --upgrade
20 changes: 20 additions & 0 deletions .github/actions/setup-rust-env/scripts/cargo-with-retry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

set -euo pipefail

TMPFILE=$(mktemp)
trap "rm -f $TMPFILE" EXIT

cargo "$@" 2>&1 | tee "$TMPFILE"
EXIT_CODE=${PIPESTATUS[0]}

if grep -qE "502 Bad Gateway|503 Service Unavailable|cache storage failed|dns error|sccache" "$TMPFILE"; then
echo ""
echo "=== sccache/cache error detected, retrying without sccache ==="
echo ""
unset RUSTC_WRAPPER
unset SCCACHE_GHA_ENABLED
cargo "$@"
else
exit $EXIT_CODE
fi
111 changes: 111 additions & 0 deletions .github/actions/wait-for-services/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
name: 'Wait for Services'
description: 'Wait for roboflow Docker Compose services to become healthy'

inputs:
timeout:
description: 'Timeout in seconds for health checks'
required: false
default: '300'

runs:
using: 'composite'
steps:
- name: Wait for services to be healthy
shell: bash
run: |
echo "Waiting for PD, TiKV, and MinIO to become healthy..."
timeout ${{ inputs.timeout }} bash -c '
for i in {1..30}; do
output=$(docker compose ps)

pd_healthy=0
tikv_healthy=0
minio_healthy=0

if echo "$output" | grep "roboflow-pd" | grep -q "healthy"; then
pd_healthy=1
else
echo " [waiting] pd"
fi

if echo "$output" | grep "roboflow-tikv" | grep -q "healthy"; then
tikv_healthy=1
else
echo " [waiting] tikv"
fi

if echo "$output" | grep "roboflow-minio" | grep -q "healthy"; then
minio_healthy=1
else
echo " [waiting] minio"
fi

echo "PD: $pd_healthy/1, TiKV: $tikv_healthy/1, MinIO: $minio_healthy/1 (attempt $i/30)"

if [ "$pd_healthy" -eq 1 ] && [ "$tikv_healthy" -eq 1 ] && [ "$minio_healthy" -eq 1 ]; then
echo "All services are healthy!"
docker compose ps
exit 0
fi

sleep 10
done

echo "Services not healthy after timeout"
docker compose ps
echo ""
echo "=== PD logs ==="
docker compose logs pd --tail 60 || true
echo ""
echo "=== TiKV logs ==="
docker compose logs tikv --tail 60 || true
echo ""
echo "=== MinIO logs ==="
docker compose logs minio --tail 60 || true
exit 1
'

- name: Verify service connectivity
shell: bash
run: |
echo "Verifying PD connectivity..."
for i in {1..30}; do
if curl -sf http://localhost:2379/health > /dev/null 2>&1 && curl -sf http://pd:2379/health > /dev/null 2>&1; then
echo "PD is healthy"
break
fi
if [ "$i" -eq 30 ]; then
echo "PD failed readiness checks" >&2
docker compose logs pd --tail 100 || true
exit 1
fi
sleep 2
done

echo "Verifying TiKV connectivity..."
for i in {1..30}; do
if curl -sf http://localhost:20180/status > /dev/null 2>&1; then
echo "TiKV is healthy"
break
fi
if [ "$i" -eq 30 ]; then
echo "TiKV failed readiness checks" >&2
docker compose logs tikv --tail 100 || true
exit 1
fi
sleep 2
done

echo "Verifying MinIO connectivity..."
for i in {1..30}; do
if curl -sf http://localhost:9000/minio/health/ready > /dev/null 2>&1; then
echo "MinIO is healthy"
break
fi
if [ "$i" -eq 30 ]; then
echo "MinIO failed readiness checks" >&2
docker compose logs minio --tail 100 || true
exit 1
fi
sleep 2
done
Loading
Loading