ci: enforce clean git trees across workflows #507
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Go - Continuous Deployment | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/go-cd.yml | |
| - .github/actions/ensure-clean-tree/action.yml | |
| - .github/workflows/install-shared-dependencies/action.yml | |
| - .github/workflows/install-engine/action.yml | |
| - .github/json_matrices/** | |
| push: | |
| tags: | |
| - "v*.*" | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: "The release version of GLIDE, formatted as v*.*.* or v*.*.*-rc*" | |
| required: true | |
| pkg_go_dev_publish: | |
| description: "Publish to pkg.go.dev" | |
| required: true | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: go-cd-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| env: | |
| BASE_GO_VERSION: "1.22" | |
| jobs: | |
| load-platform-matrix: | |
| if: github.repository_owner == 'valkey-io' | |
| runs-on: ubuntu-latest | |
| environment: AWS_ACTIONS | |
| outputs: | |
| PLATFORM_MATRIX: ${{ steps.load-platform-matrix.outputs.PLATFORM_MATRIX }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: load-platform-matrix | |
| id: load-platform-matrix | |
| shell: bash | |
| run: | | |
| # Filter entries with pkg_go_dev in PACKAGE_MANAGERS and replace "ephemeral" with "persistent" in RUNNER | |
| export PLATFORM_MATRIX=$(jq 'map( | |
| select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["pkg_go_dev"]))) | |
| | .RUNNER = ( | |
| if (.RUNNER | type == "array") | |
| then (.RUNNER | map(if . == "ephemeral" then "persistent" else . end)) | |
| else (if .RUNNER == "ephemeral" then "persistent" else .RUNNER end) | |
| end | |
| ) | |
| )' < .github/json_matrices/build-matrix.json | jq -c .) | |
| echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT | |
| validate-release-version: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| RELEASE_VERSION: ${{ steps.release-tag.outputs.RELEASE_VERSION }} | |
| env: | |
| INPUT_VERSION: ${{ github.event.inputs.version }} | |
| TAG_VERSION: ${{ github.ref_name }} | |
| steps: | |
| - name: Checkout for tag check | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Validate version agaisnt tags | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| git fetch --tags | |
| if git tag | grep -q "^go/${{ env.INPUT_VERSION }}$"; then | |
| echo "Version ${{ env.INPUT_VERSION }} already exists." | |
| exit 1 | |
| fi | |
| - name: Validate Version Against RegEx and output | |
| id: release-tag | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| R_VERSION="v0.0.0" | |
| elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| R_VERSION="${{ env.INPUT_VERSION }}" | |
| else | |
| R_VERSION="${{ env.TAG_VERSION }}" | |
| fi | |
| echo "Release version is: ${R_VERSION}" | |
| if [[ "${{ github.event_name }}" != "pull_request" ]]; then | |
| if ! echo "${R_VERSION}" | grep -Pq '^v\d+\.\d+\.\d+(-rc\d+)?$'; then | |
| echo "Version is not valid, must be v*.*.* or v*.*.*-rc*" | |
| exit 1 | |
| fi | |
| fi | |
| echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_OUTPUT | |
| create-binaries: | |
| needs: [load-platform-matrix, validate-release-version] | |
| if: github.repository_owner == 'valkey-io' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| host: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }} | |
| runs-on: ${{ matrix.host.RUNNER }} | |
| steps: | |
| - name: Setup self-hosted runner access | |
| run: | | |
| GHA_HOME=/home/ubuntu/actions-runner/_work/valkey-glide | |
| if [ -d $GHA_HOME ]; then | |
| sudo chown -R $USER:$USER $GHA_HOME | |
| sudo rm -rf $GHA_HOME | |
| mkdir -p $GHA_HOME/valkey-glide | |
| else | |
| echo "No cleaning needed" | |
| fi | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version: ${{ env.BASE_GO_VERSION }} | |
| - name: Install shared software dependencies | |
| uses: ./.github/actions/install-shared-dependencies | |
| with: | |
| os: ${{ matrix.host.OS }} | |
| target: ${{ matrix.host.CD_TARGET || matrix.host.TARGET }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build Go client | |
| working-directory: go | |
| env: | |
| RELEASE_VERSION: ${{ needs.validate-release-version.outputs.RELEASE_VERSION }} | |
| run: | | |
| make install-build-tools | |
| make build GLIDE_VERSION="${RELEASE_VERSION}" TARGET_TRIPLET="${{ matrix.host.TARGET }}" | |
| - name: Move FFI artifacts on linux | |
| if: ${{ contains(matrix.host.TARGET, 'linux') }} | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/ffi/target/release | |
| cp ffi/target/*/release/libglide_ffi.a $GITHUB_WORKSPACE/ffi/target/release/ | |
| - name: Smoke test binary | |
| run: | | |
| file ffi/target/release/libglide_ffi.a | grep -q "archive" | |
| - name: Ensure binary generation leaves tracked files unchanged | |
| uses: ./.github/actions/ensure-clean-tree | |
| with: | |
| context: Go release binary, cbindgen, and protobuf generation | |
| - name: Upload artifacts | |
| continue-on-error: true | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: ${{ matrix.host.TARGET }} | |
| path: | | |
| ffi/target/release/libglide_ffi.a | |
| go/lib.h | |
| go/internal/protobuf/ | |
| commit-auto-gen-files-with-tag: | |
| if: ${{ github.event_name == 'push' || inputs.pkg_go_dev_publish == true }} | |
| needs: [validate-release-version, create-binaries] | |
| runs-on: ubuntu-latest | |
| environment: AWS_ACTIONS | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: artifacts | |
| - name: Ensure tracked tree is clean before copying generated files | |
| uses: ./.github/actions/ensure-clean-tree | |
| with: | |
| context: Go release pre-generation baseline | |
| - name: Copy generated files to repo | |
| run: | | |
| cd artifacts | |
| # Collect all platforms for tools.go generation | |
| PLATFORMS=() | |
| for dir in */; do | |
| target_name=${dir%/} | |
| PLATFORMS+=("$target_name") | |
| mkdir -p $GITHUB_WORKSPACE/go/rustbin/${target_name} | |
| cp ${target_name}/ffi/target/release/libglide_ffi.a $GITHUB_WORKSPACE/go/rustbin/${target_name}/ | |
| # Create doc.go marker file for vendoring support | |
| # Convert platform name to valid Go package name (hyphens to underscores) | |
| package_name="rustbin_${target_name//-/_}" | |
| cat > "$GITHUB_WORKSPACE/go/rustbin/${target_name}/doc.go" << EOF | |
| // Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 | |
| // Package ${package_name} exists solely to ensure this directory is included | |
| // when using 'go mod vendor'. | |
| // | |
| // This directory contains libglide_ffi.a, the precompiled Rust FFI library | |
| // for the ${target_name} target. | |
| // | |
| // DO NOT IMPORT THIS PACKAGE. It contains no usable Go code and is never | |
| // compiled into binaries. This file only exists to make the directory a valid | |
| // Go package for vendoring purposes. | |
| // | |
| // See: https://github.com/valkey-io/valkey-glide/issues/4721 | |
| package ${package_name} | |
| EOF | |
| done | |
| # Generate tools.go to import all platform packages for vendoring | |
| cat > "$GITHUB_WORKSPACE/go/tools.go" << 'EOF' | |
| //go:build tools | |
| package glide | |
| // This file ensures rustbin platform directories are included when using 'go mod vendor'. | |
| // | |
| // This file is AUTO-GENERATED during releases. Do not edit manually. | |
| // It forces vendoring to include all platform-specific rustbin packages. | |
| // | |
| // See: https://github.com/valkey-io/valkey-glide/issues/4721 | |
| import ( | |
| EOF | |
| # Add imports for each platform | |
| for platform in "${PLATFORMS[@]}"; do | |
| echo " _ \"github.com/valkey-io/valkey-glide/go/v2/rustbin/${platform}\"" >> "$GITHUB_WORKSPACE/go/tools.go" | |
| done | |
| echo ")" >> "$GITHUB_WORKSPACE/go/tools.go" | |
| # TODO: move generation of protobuf and header file to a non-matrix step | |
| cd x86_64-unknown-linux-gnu | |
| cp go/lib.h $GITHUB_WORKSPACE/go/ | |
| mkdir -p $GITHUB_WORKSPACE/go/internal/protobuf | |
| cp go/internal/protobuf/* $GITHUB_WORKSPACE/go/internal/protobuf/ | |
| - name: Stage and validate generated release files | |
| run: | | |
| # Remove go/.gitignore from the released module. 'go mod vendor' | |
| # copies this file into the consumer's vendor/ tree, where its | |
| # rules (*.pb.go, rustbin/, lib.h) cause git to silently drop the | |
| # generated artifacts we force-add below when the consumer commits | |
| # vendor/. This tag commit is never merged back to main, so main | |
| # keeps its .gitignore for development. See issue #6437. | |
| rm -f "$GITHUB_WORKSPACE/go/.gitignore" | |
| git add -f go | |
| unexpected_paths=() | |
| while IFS= read -r path; do | |
| case "$path" in | |
| go/.gitignore|go/lib.h|go/tools.go|go/internal/protobuf/*|go/rustbin/*) | |
| ;; | |
| *) | |
| unexpected_paths+=("$path") | |
| ;; | |
| esac | |
| done < <(git diff --name-only HEAD) | |
| if (( ${#unexpected_paths[@]} > 0 )); then | |
| echo "Unexpected tracked changes in Go generated release files:" | |
| echo "Tracked status (first 200 entries):" | |
| git status --short --untracked-files=no | sed -n '1,200p' | |
| echo "Unexpected paths (first 200 entries):" | |
| printf ' %s\n' "${unexpected_paths[@]}" | sed -n '1,200p' | |
| exit 1 | |
| fi | |
| echo "Tracked status (first 200 entries):" | |
| git status --short --untracked-files=no | sed -n '1,200p' | |
| - name: Commit generated release files | |
| run: | | |
| git config user.name github-actions | |
| git config user.email github-actions@github.com | |
| git commit -m "Automated commit from GitHub Actions" | |
| - name: Ensure release commit leaves tracked tree clean | |
| uses: ./.github/actions/ensure-clean-tree | |
| with: | |
| context: Go generated release commit | |
| - name: Push generated release tag | |
| run: | | |
| RELEASE_VERSION=${{ needs.validate-release-version.outputs.RELEASE_VERSION }} | |
| git tag go/$RELEASE_VERSION | |
| git push origin go/$RELEASE_VERSION | |
| GOPROXY=proxy.golang.org go list -m github.com/valkey-io/valkey-glide/go/v2@$RELEASE_VERSION | |
| extra-post-commit-test: | |
| needs: | |
| [ | |
| load-platform-matrix, | |
| validate-release-version, | |
| commit-auto-gen-files-with-tag, | |
| ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| host: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }} | |
| # Excluding musl targets for testing for now as there are issues with the runners | |
| exclude: | |
| - host: | |
| TARGET: aarch64-unknown-linux-musl | |
| - host: | |
| TARGET: x86_64-unknown-linux-musl | |
| runs-on: ${{ matrix.host.RUNNER }} | |
| steps: | |
| - name: Setup self-hosted runner access | |
| run: | | |
| GHA_HOME=/home/ubuntu/actions-runner/_work/valkey-glide | |
| if [ -d $GHA_HOME ]; then | |
| sudo chown -R $USER:$USER $GHA_HOME | |
| sudo rm -rf $GHA_HOME | |
| mkdir -p $GHA_HOME/valkey-glide | |
| else | |
| echo "No cleaning needed" | |
| fi | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: "go/${{ needs.validate-release-version.outputs.RELEASE_VERSION }}" | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version: ${{ env.BASE_GO_VERSION }} | |
| - name: Install shared software dependencies | |
| uses: ./.github/actions/install-shared-dependencies | |
| with: | |
| os: ${{ matrix.host.OS }} | |
| target: ${{ matrix.host.TARGET }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| engine-version: "8.0" | |
| - name: Start standalone Valkey server | |
| working-directory: utils | |
| id: start-server | |
| run: | | |
| PORT=$(python3 ./cluster_manager.py start -r 0 2>&1 | grep CLUSTER_NODES | cut -d = -f 2 | cut -d , -f 1 | cut -d : -f 2) | |
| echo "PORT=$PORT" >> $GITHUB_OUTPUT | |
| - name: Ensure tracked tree is clean before benchmark module edits | |
| uses: ./.github/actions/ensure-clean-tree | |
| with: | |
| context: Go benchmark pre-tidy baseline | |
| - name: test newly released go client | |
| env: | |
| PORT: ${{ steps.start-server.outputs.PORT }} | |
| working-directory: go/benchmarks | |
| run: | | |
| validate_benchmark_changes() { | |
| unexpected_paths=() | |
| while IFS= read -r path; do | |
| case "$path" in | |
| go/benchmarks/go.mod|go/benchmarks/go.sum) | |
| ;; | |
| *) | |
| unexpected_paths+=("$path") | |
| ;; | |
| esac | |
| done < <(git -C "$GITHUB_WORKSPACE" diff --name-only HEAD) | |
| if (( ${#unexpected_paths[@]} > 0 )); then | |
| echo "Unexpected tracked changes in Go benchmark module test:" | |
| echo "Tracked status (first 200 entries):" | |
| git -C "$GITHUB_WORKSPACE" status --short --untracked-files=no | sed -n '1,200p' | |
| echo "Unexpected paths (first 200 entries):" | |
| printf ' %s\n' "${unexpected_paths[@]}" | sed -n '1,200p' | |
| exit 1 | |
| fi | |
| } | |
| # change go/benchmarks/go.mod on the fly | |
| RELEASE_VERSION=${{ needs.validate-release-version.outputs.RELEASE_VERSION }} | |
| if [[ "${{ matrix.host.OS }}" == "macos" ]]; then | |
| sed -i '' '\|replace github\.com/valkey-io/valkey-glide/go/v2|d' go.mod | |
| sed -i '' "s|github\.com/valkey-io/valkey-glide/go/v2 v.*|github.com/valkey-io/valkey-glide/go/v2 $RELEASE_VERSION|g" go.mod | |
| else | |
| sed -i '\|replace github\.com/valkey-io/valkey-glide/go/v2|d' go.mod | |
| sed -i "s|github\.com/valkey-io/valkey-glide/go/v2 v.*|github.com/valkey-io/valkey-glide/go/v2 $RELEASE_VERSION|g" go.mod | |
| fi | |
| go mod tidy | |
| validate_benchmark_changes | |
| go run . -minimal -clients glide -concurrentTasks 10 -port ${{ env.PORT }} | |
| validate_benchmark_changes |