Redesign CoreStore for bounded distributed operation #513
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Anvil CI | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| packages: read | |
| env: | |
| # Self-hosted runners keep this directory between jobs. Sharing Cargo's | |
| # target directory avoids rebuilding the same workspace dependencies in the | |
| # image, unit, server-core and Docker-gate stages. | |
| CARGO_TARGET_DIR: /home/github-runner/.cache/anvil/cargo-target | |
| jobs: | |
| build-image: | |
| name: Build Anvil test image | |
| runs-on: [self-hosted, linux, x64, worka-builder-2] | |
| timeout-minutes: 45 | |
| env: | |
| ANVIL_DOCKER_PLATFORM: linux/amd64 | |
| ANVIL_IMAGE: anvil:test-amd64-${{ github.run_id }}-${{ github.run_attempt }} | |
| ANVIL_USE_NATIVE_CARGO: "1" | |
| ANVIL_BUILD_PROFILE: ci | |
| ANVIL_INCLUDE_TEST_TOOLS: "1" | |
| ANVIL_RUNTIME_BASE: ubuntu:24.04 | |
| CARGO_BUILD_JOBS: "8" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| clang \ | |
| cmake \ | |
| libclang-dev \ | |
| pkg-config \ | |
| libssl-dev \ | |
| protobuf-compiler \ | |
| ripgrep | |
| - name: Cache Cargo dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-v2-pr-image-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-v2-pr-image- | |
| ${{ runner.os }}-cargo-v2- | |
| - name: Build Docker image from host-built binaries | |
| run: ./scripts/build-image.sh | |
| - name: Smoke check image binaries and test tooling | |
| run: | | |
| timeout 30s docker run --platform linux/amd64 --rm "$ANVIL_IMAGE" anvil-server --version | |
| timeout 30s docker run --platform linux/amd64 --rm "$ANVIL_IMAGE" anvil --version | |
| timeout 30s docker run --platform linux/amd64 --rm "$ANVIL_IMAGE" anvil-admin --version | |
| timeout 30s docker run --platform linux/amd64 --rm "$ANVIL_IMAGE" ip -Version | |
| - name: Export Docker image artifact | |
| run: docker save "$ANVIL_IMAGE" -o "${{ runner.temp }}/anvil-test-image-amd64.tar" | |
| - name: Upload Docker image artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: anvil-test-image-amd64 | |
| path: ${{ runner.temp }}/anvil-test-image-amd64.tar | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 1 | |
| - name: Remove local Docker image | |
| if: always() | |
| run: | | |
| if docker image inspect "$ANVIL_IMAGE" >/dev/null 2>&1; then | |
| docker image rm "$ANVIL_IMAGE" || echo "::warning::Failed to remove $ANVIL_IMAGE" | |
| fi | |
| static: | |
| name: Static, docs, and packaging gates | |
| runs-on: [self-hosted, linux, x64, worka-builder-2] | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| clang \ | |
| cmake \ | |
| libclang-dev \ | |
| pkg-config \ | |
| libssl-dev \ | |
| protobuf-compiler \ | |
| ripgrep | |
| - name: Cache Cargo dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-v2-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-v2- | |
| - name: Install Fission CLI | |
| run: | | |
| if ! command -v fission >/dev/null 2>&1; then | |
| cargo install cargo-fission --version 0.4.1 --locked | |
| fi | |
| - name: Run static release gates | |
| run: ./scripts/release-gates.sh static | |
| rust-unit: | |
| name: Rust unit and client tests | |
| needs: [build-image, static] | |
| runs-on: [self-hosted, linux, x64, worka-builder-2] | |
| timeout-minutes: 120 | |
| env: | |
| CARGO_BUILD_JOBS: "8" | |
| CARGO_PROFILE_DEV_DEBUG: "0" | |
| CARGO_PROFILE_TEST_DEBUG: "0" | |
| ANVIL_GATE_STEP_TIMEOUT_SECONDS: "5400" | |
| ANVIL_RUST_TEST_THREADS: "8" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| clang \ | |
| cmake \ | |
| libclang-dev \ | |
| pkg-config \ | |
| libssl-dev \ | |
| protobuf-compiler \ | |
| ripgrep | |
| - name: Cache Cargo dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-v2-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-v2- | |
| - name: Run Rust unit release gates | |
| run: ./scripts/release-gates.sh rust | |
| coremeta-performance: | |
| name: CoreMeta ordered-access performance gate | |
| needs: rust-unit | |
| runs-on: [self-hosted, linux, x64, worka-builder-2] | |
| timeout-minutes: 60 | |
| env: | |
| CARGO_BUILD_JOBS: "8" | |
| ANVIL_MACHINE_CLASS: "worka-builder-2-linux-x64" | |
| ANVIL_GATE_STEP_TIMEOUT_SECONDS: "3300" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| clang \ | |
| cmake \ | |
| libclang-dev \ | |
| pkg-config \ | |
| libssl-dev \ | |
| protobuf-compiler | |
| - name: Cache Cargo dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-v2-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-v2- | |
| - name: Run quick CoreMeta performance gate | |
| run: ./scripts/release-gates.sh perf-quick | |
| - name: Upload CoreMeta performance evidence | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coremeta-performance-quick | |
| path: target/anvil/perf/coremeta/quick/ | |
| if-no-files-found: error | |
| retention-days: 14 | |
| server-core: | |
| name: Server core integration tests | |
| needs: coremeta-performance | |
| runs-on: [self-hosted, linux, x64, worka-builder-2] | |
| timeout-minutes: 60 | |
| env: | |
| CARGO_BUILD_JOBS: "8" | |
| CARGO_PROFILE_DEV_DEBUG: "0" | |
| CARGO_PROFILE_TEST_DEBUG: "0" | |
| ANVIL_RUST_TEST_THREADS: "8" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| clang \ | |
| cmake \ | |
| libclang-dev \ | |
| pkg-config \ | |
| libssl-dev \ | |
| protobuf-compiler \ | |
| ripgrep | |
| - name: Cache Cargo dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-v2-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-v2- | |
| - name: Prebuild server integration test binaries | |
| run: cargo test --no-run -p anvil-server --tests | |
| - name: Run server core integration gates | |
| run: ./scripts/release-gates.sh server-core | |
| docker-integration: | |
| name: Docker integration tests (${{ matrix.group }}) | |
| needs: [build-image, server-core] | |
| runs-on: [self-hosted, linux, x64, worka-builder-2] | |
| timeout-minutes: 120 | |
| env: | |
| CARGO_BUILD_JOBS: "8" | |
| CARGO_PROFILE_DEV_DEBUG: "0" | |
| CARGO_PROFILE_TEST_DEBUG: "0" | |
| ANVIL_DOCKER_TEST_THREADS: "2" | |
| ANVIL_IMAGE: anvil:test-amd64-${{ github.run_id }}-${{ github.run_attempt }} | |
| ANVIL_DOCKER_TEST_PROJECT: anvil-ci-${{ matrix.group }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| ANVIL_GATE_STEP_TIMEOUT_SECONDS: "5400" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| group: [docker-auth, docker-storage, docker-index, docker-mesh] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| clang \ | |
| cmake \ | |
| libclang-dev \ | |
| pkg-config \ | |
| libssl-dev \ | |
| protobuf-compiler | |
| - name: Cache Cargo dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-v2-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-v2- | |
| - name: Prebuild Docker integration test binaries | |
| run: cargo test --no-run -p anvil-server -p anvil-storage-cli --tests | |
| - name: Download amd64 Docker image artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: anvil-test-image-amd64 | |
| path: ${{ runner.temp }}/anvil-image-amd64 | |
| - name: Load amd64 Docker image | |
| run: docker load -i "${{ runner.temp }}/anvil-image-amd64/anvil-test-image-amd64.tar" | |
| - name: Run Docker integration release gates | |
| run: ./scripts/release-gates.sh ${{ matrix.group }} | |
| - name: Clean up Docker integration resources | |
| if: always() | |
| run: ./scripts/release-gates.sh docker-cleanup |