Merge branch 'apache:master' into export_gcs_options

.asf.yaml

@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.55.0: {}
     release-2.54.0: {}
     release-2.53.0: {}
     release-2.52.0: {}

.github/REVIEWERS.yml

@@ -27,12 +27,12 @@ labels:
       - youngoli
   - name: Python
     reviewers:
-      - AnandInguva
       - damccorm
       - jrmccluskey
       - tvalentyn
       - riteshghorse
       - liferoad
+      - shunping
     exclusionList: []
   - name: Java
     reviewers:
@@ -51,6 +51,7 @@ labels:
       - ahmedabu98
       - bvolpato
       - damondouglas
+      - shunping
     exclusionList: []
   - name: spanner
     reviewers:
@@ -79,7 +80,6 @@ labels:
     exclusionList: []
 fallbackReviewers:
   - Abacn
-  - AnandInguva
   - chamikaramj
   - damccorm
   - damondouglas
@@ -89,4 +89,5 @@ fallbackReviewers:
   - lostluck
   - riteshghorse
   - robertwb
+  - shunping
   - tvalentyn

.github/actions/setup-k8s-access/action.yml

@@ -34,7 +34,10 @@ inputs:
     description: "Zone of the cluster to be created"
     required: true
     default: "us-central1-a"
-
+  remove_finalizer:
+    description: "Remove finalizers from the cluster"
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -70,4 +73,7 @@ runs:
       main: echo "Post Cleanup"
       post: |
         echo "Post Cleanup"
+        if [ -n "${{ inputs.remove_finalizer }}" ]; then
+          kubectl patch ${{ inputs.remove_finalizer }} -p '[{"op": "remove", "path": "/metadata/finalizers"}]' --type=json
+        fi
         kubectl delete namespace ${{ steps.replace_namespace.outputs.TEST_NAMESPACE }}

.github/build.gradle

@@ -50,7 +50,7 @@ task check {
 
         // precommit and postcommit should triggered by this specific file
         // this is to ensure not missing test during release branch verification
-        if (paths != null && !paths.contains('release/trigger_all_tests.json')) {
+        if (paths != null && !paths.contains('release/trigger_all_tests.json') && !fname.toLowerCase().contains('sickbay')) {
           errors.add("Error validating ${fname}: " +
                   "Please add 'release/trigger_all_tests.json' to the trigger path")
           return

.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json

@@ -0,0 +1,3 @@
+{
+    "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_DataflowV1.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Python_Examples_Direct.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json

@@ -0,0 +1,4 @@
+
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json

@@ -1 +1,3 @@
-
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_Python_Xlang_IO_Dataflow.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_TransformService_Direct.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/trigger_files/beam_PostCommit_XVR_Flink.json

@@ -0,0 +1,3 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

.github/workflows/beam_CancelStaleDataflowJobs.yml

@@ -71,6 +71,8 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        with:
+          disable-cache: true
       - name: Authenticate on GCP
         id: auth
         uses: google-github-actions/auth@v1

.github/workflows/beam_CleanUpDataprocResources.yml

@@ -17,7 +17,7 @@ name: Cleanup Dataproc Resources
 
 on:
   schedule:
-    - cron: '0 */6 * * *'
+    - cron: '0 0 * * *'
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

.github/workflows/beam_CleanUpGCPResources.yml

@@ -71,6 +71,8 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        with:
+          disable-cache: true
       - name: Authenticate on GCP
         id: auth
         uses: google-github-actions/setup-gcloud@v0

.github/workflows/beam_CleanUpPrebuiltSDKImages.yml

@@ -52,7 +52,7 @@ jobs:
   beam_CleanUpPrebuiltSDKImages:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 100
+    timeout-minutes: 180
     strategy:
       matrix: 
         job_name: [beam_CleanUpPrebuiltSDKImages]
@@ -71,12 +71,18 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        with:
+          disable-cache: true
       - name: Authenticate on GCP
         id: auth
         uses: google-github-actions/auth@v1
         with:
           credentials_json: ${{ secrets.GCP_SA_KEY }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
+      - name: GCloud Docker credential helper
+        run: |
+          gcloud auth configure-docker gcr.io && \
+          gcloud auth configure-docker us.gcr.io
       - name: run remove stale sdk container images
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:

.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml

@@ -17,7 +17,7 @@ name: LoadTests Python Combine Dataflow Streaming
 
 on:
   schedule:
-    - cron: '50 15 * * *'
+    - cron: '50 5 * * *'
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

.github/workflows/beam_MetricsCredentialsRotation.yml

@@ -17,7 +17,7 @@ name: Rotate Metrics Cluster Credentials
 
 on:
   schedule:
-    - cron: '0 2 1 * *'
+    - cron: '0 1 1 * *'
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -50,7 +50,7 @@ jobs:
   beam_MetricsCredentialsRotation:
     if: |
       github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'schedule'
+      (github.event_name == 'schedule' && github.repository == 'apache/beam')
     runs-on: [self-hosted, ubuntu-20.04, main]
     timeout-minutes: 100
     name: ${{ matrix.job_name }}
@@ -95,4 +95,4 @@ jobs:
           to: dev@beam.apache.org
           from: gactions@beam.apache.org
           body: |
-            Something went wrong during the automatic credentials rotation for Metrics Cluster, performed at ${{ env.date }}. It may be necessary to check the state of the cluster certificates. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_MetricsCredentialsRotation.yml \n * Cluster URL: https://pantheon.corp.google.com/kubernetes/clusters/details/us-central1-a/metrics/details?mods=dataflow_dev&project=apache-beam-testing
+            Something went wrong during the automatic credentials rotation for Metrics Cluster, performed at ${{ env.date }}. It may be necessary to check the state of the cluster certificates. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_MetricsCredentialsRotation.yml \n * Cluster URL: https://pantheon.corp.google.com/kubernetes/clusters/details/us-central1-a/metrics/details?mods=dataflow_dev&project=apache-beam-testing

.github/workflows/beam_PerformanceTests_SingleStoreIO.yml

@@ -83,6 +83,7 @@ jobs:
         with:
           cluster_name: io-datastores
           k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }}
+          remove_finalizer:  memsqlclusters.memsql.com/sdb-cluster
       - name: Install Singlestore operator
         run: | 
           kubectl apply -f ${{github.workspace}}/.test-infra/kubernetes/singlestore/sdb-rbac.yaml

.github/workflows/beam_Playground_CI_Nightly.yml

@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Playground CI Nightly
+
+on:
+  schedule:
+    - cron: '20 */12 * * *'
+  workflow_dispatch:
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: read
+  checks: read
+  contents: read
+  deployments: read
+  id-token: none
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}'
+  cancel-in-progress: true
+
+env:
+  GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+  BEAM_ROOT_DIR: /runner/_work/beam/beam
+
+
+jobs:
+  beam_Playground_CI_Nightly:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'pull_request_target' ||
+      (github.event_name == 'schedule' && github.repository == 'apache/beam')
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    name: "beam_Playground_CI_Nightly"
+    strategy:
+      matrix: 
+        sdk: ["python", "java", "go"]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment-action
+      - name: Install Python 3.8
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.8'
+      - name: Install requirements
+        run: |
+          cd $BEAM_ROOT_DIR/playground/infrastructure
+          pip install -r requirements.txt
+      - name: Get Beam latest release
+        run: |
+          BEAM_VERSION=$(curl -s https://api.github.com/repos/apache/beam/releases/latest | jq -r '.tag_name')
+          echo "BEAM_VERSION=${BEAM_VERSION#v}" >> $GITHUB_ENV
+      - name: Build PYTHON base
+        if: ${{ matrix.sdk == 'python' }}
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:container:py310:docker -Pdocker-tag=nightly -Pdocker-pull-licences=true
+      - name: Build SDK container
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :playground:backend:containers:${{ matrix.sdk }}:docker -Pdocker-tag=nightly -Psdk-tag=$BEAM_VERSION -Pdocker-pull-licences=true
+      - name: Run SDK container
+        run: |
+          CONTAINER_ID=$(docker run -d -e PROTOCOL_TYPE=TCP apache/beam_playground-backend-${{ matrix.sdk }}:nightly)
+          echo "container_id=$CONTAINER_ID" >> $GITHUB_ENV
+      - name: Get Container IP
+        run: | 
+          CONTAINER_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ env.container_id }})
+          echo "container_ip=$CONTAINER_IP" >> $GITHUB_ENV
+      - name: Run CI
+        env: 
+          SERVER_ADDRESS: ${{ env.container_ip }}:8080
+          BEAM_EXAMPLE_CATEGORIES: ${{ env.BEAM_ROOT_DIR }}/playground/categories.yaml
+          SDK: ${{ matrix.sdk }}
+        run: |
+          cd $BEAM_ROOT_DIR/playground/infrastructure
+          python ci_cd.py --step CI --sdk SDK_${SDK^^} --origin PG_EXAMPLES --subdirs ./learning/katas ./examples ./sdks

.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml

@@ -85,6 +85,25 @@ jobs:
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:java:io:google-cloud-platform:bigQueryEarlyRolloutIntegrationTest
+      - name: Generate Date
+        if: failure()
+        run: |
+          date=$(date -u +"%Y-%m-%d")
+          echo "date=$date" >> $GITHUB_ENV
+      - name: Send email
+        uses: dawidd6/action-send-mail@v3
+        if: failure()
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          secure: true
+          username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
+          password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
+          subject: BigQueryEarlyRollout Beam Failure (${{ env.date }}) - investigate and escalate quickly
+          to: datapls-plat-team@google.com # Team at Google responsible for escalating BQ failures
+          from: gactions@beam.apache.org
+          body: |
+            PostCommit Java BigQueryEarlyRollout failed on ${{ env.date }}. This test monitors BigQuery rollouts impacting Beam and should be escalated immediately if a real issue is encountered to pause further rollouts. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}
@@ -97,4 +116,4 @@ jobs:
         with:
           commit: '${{ env.prsha || env.GITHUB_SHA }}'
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
-          files: '**/build/test-results/**/*.xml'
+          files: '**/build/test-results/**/*.xml'

.github/workflows/beam_PostCommit_Java_DataflowV1.yml

@@ -19,7 +19,7 @@ name: PostCommit Java Dataflow V1
 
 on:
   schedule:
-    - cron: '30 3/6 * * *'
+    - cron: '30 3/8 * * *'
   pull_request_target:
     paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_DataflowV1.json']
   workflow_dispatch:
@@ -54,7 +54,7 @@ jobs:
   beam_PostCommit_Java_DataflowV1:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 240
+    timeout-minutes: 360
     strategy:
       matrix:
         job_name: [beam_PostCommit_Java_DataflowV1]