Merge branch 'protocolbuffers:main' into unset-by-index

.github/BUILD.bazel

@@ -0,0 +1,27 @@
+# This information is extracted from the MacOS runner specs located at:
+# https://github.com/actions/runner-images/blob/win19/20230129.2/images/macos/macos-12-Readme.md
+xcode_version(
+    name = "version14_2_14C18",
+    version = "14.2.14C18",
+    aliases = ["14C18"],
+    default_ios_sdk_version = "16.2",
+    default_tvos_sdk_version = "16.1",
+    default_macos_sdk_version = "13.1",
+    default_watchos_sdk_version = "9.1",
+)
+
+xcode_version(
+    name = "version14_1_0_14B47b",
+    version = "14.1.0.14B47b",
+    aliases = ["14B47b"],
+    default_ios_sdk_version = "16.1",
+    default_tvos_sdk_version = "16.1",
+    default_macos_sdk_version = "13.0",
+    default_watchos_sdk_version = "9.1",
+)
+
+xcode_config(
+    name = "host_xcodes",
+    versions = [":version14_2_14C18", ":version14_1_0_14B47b"],
+    default = ":version14_1_0_14B47b",
+)

.github/CODEOWNERS

@@ -34,3 +34,4 @@
 /kokoro/ @protocolbuffers/protobuf-btr
 /third_party/ @protocolbuffers/protobuf-btr
 *.bazel @protocolbuffers/protobuf-btr
+/.github/ @protocolbuffers/protobuf-btr

.github/actions/bash/action.yml

@@ -0,0 +1,27 @@
+name: 'Non-Bazel Bash Run'
+description: 'Run a bash script for Protobuf CI testing with a non-Bazel build system'
+inputs:
+  credentials:
+    required: true
+    description: "The GCP credentials to use for reading the docker image"
+    type: string
+  command:
+    required: true
+    description: A command to run in the docker image
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup Runner
+      uses: ./.github/actions/internal/setup-runner
+
+    - name: Update stale files using Bazel
+      uses: ./.github/actions/bazel
+      with:
+        credentials: ${{ inputs.credentials }}
+        bazel-cache: regenerate-stale-files
+        bash: ./regenerate_stale_files.sh $BAZEL_FLAGS
+
+    - name: Run
+      shell: bash
+      run: ${{ inputs.command }}

.github/actions/bazel-docker/action.yml

@@ -0,0 +1,75 @@
+name: 'Docker Bazel Run'
+description: 'Run a Bazel-based docker image for Protobuf CI testing'
+inputs:
+  credentials:
+    required: true
+    description: "The GCP credentials to use for reading the docker image"
+    type: string
+  image:
+    required: false
+    default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
+    description: "The docker image to use"
+    type: string
+  bazel-cache:
+    required: true
+    description: >
+      A unique path for the Bazel cache.  This will trigger the generation
+      of a BAZEL_CACHE environment variable inside the container that provides
+      the appropriate flags for any bazel command.
+    type: string
+  bazel:
+    required: false
+    description: "The Bazel command to run"
+    type: string
+  bash:
+    required: false
+    description: "A bash command to run.  $BAZEL_FLAGS will be available to use for bazel runs."
+    type: string
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Authenticate
+      id: auth
+      uses: ./.github/actions/internal/gcloud-auth
+      with:
+        credentials: ${{ inputs.credentials }}
+
+    - name: Setup Runner
+      uses: ./.github/actions/internal/setup-runner
+
+    - name: Setup Bazel
+      id: bazel
+      uses: ./.github/actions/internal/bazel-setup
+      with:
+        credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
+        bazel-cache: ${{ inputs.bazel-cache }}
+
+    - name: Hook up repository Cache
+      shell: bash
+      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache='/workspace/${{ env.REPOSITORY_CACHE_PATH }}'" >> $GITHUB_ENV
+
+    - name: Validate inputs
+      if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
+      shell: bash
+      run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
+
+    - name: Run Bash Docker
+      uses: ./.github/actions/internal/docker-run
+      if: ${{ inputs.bash }}
+      with:
+        image: ${{ inputs.image }}
+        run-flags: --entrypoint "/bin/bash"
+        command: -l -c "${{ inputs.bash }}"
+
+    - name: Run Bazel Docker
+      uses: ./.github/actions/internal/docker-run
+      if: ${{ !inputs.bash }}
+      with:
+        image: ${{ inputs.image }}
+        command: ${{ inputs.bazel }} ${{ env.BAZEL_FLAGS }}
+
+    - name: Save Bazel repository cache
+      # Only allow repository cache updates during post-submits.
+      if: ${{ github.event_name == 'push' }}
+      uses: ./.github/actions/internal/repository-cache-save

.github/actions/bazel/action.yml

@@ -0,0 +1,111 @@
+name: 'Docker Bazel Run'
+description: 'Run a Bazel-based docker image for Protobuf CI testing'
+inputs:
+  credentials:
+    required: true
+    description: The GCP credentials to use for reading the docker image
+    type: string
+  bazel-cache:
+    required: true
+    description: >
+      A unique path for the Bazel cache.  This will trigger the generation
+      of a BAZEL_CACHE environment variable inside the container that provides
+      the appropriate flags for any bazel command.
+    type: string
+  version:
+    required: false
+    description: A pinned Bazel version to use
+    default: '5.1.1'
+    type: string
+  bazel:
+    required: false
+    description: The Bazel command to run
+    type: string
+  bash:
+    required: false
+    description: >
+      A bash command to run.  $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be
+      available to use for bazel runs.
+    type: string
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Authenticate
+      id: auth
+      uses: ./.github/actions/internal/gcloud-auth
+      with:
+        credentials: ${{ inputs.credentials }}
+
+    - name: Setup Runner
+      uses: ./.github/actions/internal/setup-runner
+
+    - name: Setup Bazel
+      id: bazel
+      uses: ./.github/actions/internal/bazel-setup
+      with:
+        credentials-file: ${{ steps.auth.outputs.credentials-file }}
+        bazel-cache: ${{ inputs.bazel-cache }}
+
+    - name: Get Linux bazelisk path
+      if: runner.os == 'Linux'
+      shell: bash
+      run: echo "BAZELISK_PATH=~/.cache/bazelisk" >> $GITHUB_ENV
+
+    - name: Get MacOS bazelisk path
+      if: runner.os == 'macOS'
+      shell: bash
+      run: echo "BAZELISK_PATH=~/Library/Caches/bazelisk" >> $GITHUB_ENV
+
+    - name: Get Windows bazelisk path
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "BAZELISK_PATH=$LOCALAPPDATA\bazelisk" >> $GITHUB_ENV
+
+    - name: Cache Bazelisk
+      if: ${{ github.event_name == 'push' }}
+      uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
+      with:
+        path: ${{ env.BAZELISK_PATH }}
+        key: bazel-${{ runner.os }}-${{ inputs.version }}
+
+    - name: Restore Bazelisk
+      if: ${{ github.event_name != 'push' }}
+      uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
+      with:
+        path: ${{ env.BAZELISK_PATH }}
+        key: bazel-${{ runner.os }}-${{ inputs.version }}
+
+    - name: Hook up repository Cache
+      shell: bash
+      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache=$(pwd)/${{ env.REPOSITORY_CACHE_PATH }}" >> $GITHUB_ENV
+
+    - name: Validate inputs
+      if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
+      shell: bash
+      run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
+
+    - name: Pin Bazel version
+      shell: bash
+      run: echo "USE_BAZEL_VERSION=${{ inputs.version }}" >> $GITHUB_ENV
+
+    - name: Output Bazel version
+      shell: bash
+      run: bazelisk version
+
+    - name: Run Bash
+      if: ${{ inputs.bash }}
+      run: ${{ inputs.bash }}
+      shell: bash
+
+    - name: Run Bazel
+      if: ${{ !inputs.bash }}
+      run: >-
+        bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }}
+        ${{ inputs.bazel }} $BAZEL_FLAGS
+      shell: bash
+
+    - name: Save Bazel repository cache
+      # Only allow repository cache updates during post-submits.
+      if: ${{ github.event_name == 'push' }}
+      uses: ./.github/actions/internal/repository-cache-save

.github/actions/ccache/action.yml

@@ -0,0 +1,65 @@
+name: 'CCache Setup'
+description: 'Run a Bazel-based docker image for Protobuf CI testing'
+inputs:
+  cache-prefix:
+    required: true
+    description: A unique prefix to prevent cache pollution
+    type: string
+  support-modules:
+    required: false
+    description: Whether or not we need to support modules.  This can result in extra cache misses.
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup ccache on Windows
+      if: ${{ runner.os == 'Windows' }}
+      uses: ./.github/actions/internal/ccache-setup-windows
+    - name: Setup ccache on Mac
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: brew install ccache
+
+    - name: Setup fixed path ccache caching
+      uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
+      with:
+        path: .ccache
+        # Always push to a cache key unique to this commit.
+        key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }}
+        # Select a cache to restore from with the follow order of preference:
+        # 1) The exact same commit we're running over
+        # 2) The latest cache from the current ref branch
+        # 3) The latest push to the base ref of a pull request
+        restore-keys: |
+          ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }}
+          ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }}
+          ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }}
+
+    - name: Configure ccache environment variables
+      shell: bash
+      run: |
+        echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
+        echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV
+        echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV
+        echo "CCACHE_MAXSIZE=600M" >> $GITHUB_ENV
+        echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV
+        echo "CCACHE_DIRECT=true" >> $GITHUB_ENV
+        echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache $CCACHE_CMAKE_FLAGS" >> $GITHUB_ENV
+
+    - name: Enable module support
+      if: ${{ inputs.support-modules }}
+      shell: bash
+      run: |
+        echo "CCACHE_SLOPPINESS=$CCACHE_SLOPPINESS,modules" >> $GITHUB_ENV
+        echo "CCACHE_DEPEND=true" >> $GITHUB_ENV
+
+    - name: Zero out ccache
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: ccache -z
+
+    - name: Zero out ccache
+      if: ${{ runner.os == 'Windows' }}
+      shell: pwsh
+      run: ${{ github.workspace }}\ccache.exe -z

.github/actions/cross-compile-protoc/action.yml

@@ -0,0 +1,38 @@
+name: 'Cross-compile protoc'
+description: 'Produces a cross-compiled protoc binary for a target architecture'
+inputs:
+  credentials:
+    required: true
+    description: The GCP credentials to use for reading the docker image
+    type: string
+  architecture:
+    required: true
+    description: The target architecture to build for
+    type: string
+outputs:
+  protoc:
+    description: "Cross-compiled protoc location.  Also output to $PROTOC"
+    value: ${{ steps.output.outputs.protoc }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Cross compile protoc for ${{ inputs.architecture }}
+      uses: ./.github/actions/bazel-docker
+      with:
+        credentials: ${{ inputs.credentials }}
+        bazel-cache: xcompile-protoc/${{ inputs.architecture }}
+        bash: |
+          bazel build //:protoc --config=${{ inputs.architecture }} $BAZEL_FLAGS
+          cp bazel-bin/protoc .
+
+    - name: Set protoc environment variable
+      shell: bash
+      run: echo "PROTOC=protoc-${{ inputs.architecture }}" >> $GITHUB_ENV
+
+    - name: Extract binary
+      id: output
+      shell: bash
+      run: |
+        mv protoc $PROTOC
+        echo "protoc=$PROTOC" >> $GITHUB_OUTPUT