Add spanner module build override based on changeset for cicd

.github/workflows/java-pr.yml

@@ -27,6 +27,14 @@ on:
       # This will make it easier to verify action changes don't break anything.
       - '.github/actions/setup-env/*'
       - '.github/workflows/java-pr.yml'
+      # Exclude spanner paths from global run
+      - '!v2/datastream-to-spanner/**'
+      - '!v2/spanner-common/**'
+      - '!v2/spanner-change-streams-to-sharded-file-sink/**'
+      - '!v2/gcs-to-sourcedb/**'
+      - '!v2/spanner-migrations-sdk/**'
+      - '!v2/spanner-custom-shard/**'
+      - '!v2/sourcedb-to-spanner/**'
   schedule:
     - cron: "0 */12 * * *"
   workflow_dispatch:

.github/workflows/spanner-pr.yml

@@ -0,0 +1,221 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Checks that are intended to run on PRs containing Java code.
+
+name: Spanner PR
+
+on:
+  pull_request:
+    branches:
+    - 'main'
+    paths:
+    # Include spanner paths only
+    - 'v2/datastream-to-spanner/**'
+    - 'v2/spanner-common/**'
+    - 'v2/spanner-change-streams-to-sharded-file-sink/**'
+    - 'v2/gcs-to-sourcedb/**'
+    - 'v2/spanner-migrations-sdk/**'
+    - 'v2/spanner-custom-shard/**'
+    - 'v2/sourcedb-to-spanner/**'
+  schedule:
+  - cron: "0 */12 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: java-pr-${{ github.event.issue.number || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
+
+permissions: read-all
+
+jobs:
+  spotless_check:
+    name: Spotless
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      with:
+        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Spotless
+      run: |
+        ./cicd/run-spotless \
+        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
+        --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/"
+  checkstyle_check:
+    name: Checkstyle
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      with:
+        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Checkstyle
+      run: |
+        ./cicd/run-checkstyle \
+        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
+        --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/"
+  java_build:
+    name: Build
+    timeout-minutes: 60
+    runs-on: [self-hosted, it]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      with:
+        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Build
+      run: |
+        ./cicd/run-build \
+        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
+        --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/"
+    - name: Cleanup Java Environment
+      uses: ./.github/actions/cleanup-java-env
+  java_unit_tests:
+    name: Unit Tests
+    needs: [java_build]
+    timeout-minutes: 60
+    runs-on: [self-hosted, it]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      with:
+        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Unit Tests
+      run: |
+        ./cicd/run-unit-tests \
+        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
+        --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/"
+    - name: Upload Unit Tests Report
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      if: always() # always run even if the previous step fails
+      with:
+        name: surefire-test-results
+        path: '**/surefire-reports/TEST-*.xml'
+        retention-days: 1
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4.0.1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        slug: GoogleCloudPlatform/DataflowTemplates
+        files: 'target/site/jacoco-aggregate/jacoco.xml'
+    - name: Cleanup Java Environment
+      uses: ./.github/actions/cleanup-java-env
+  java_integration_smoke_tests_templates:
+    name: Dataflow Templates Integration Smoke Tests
+    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
+    timeout-minutes: 60
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [self-hosted, it]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      with:
+        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Integration Smoke Tests
+      run: |
+        ./cicd/run-it-smoke-tests \
+        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
+        --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" \
+        --it-region="us-central1" \
+        --it-project="cloud-teleport-testing" \
+        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+        --it-private-connectivity="datastream-private-connect-us-central1"
+    - name: Upload Smoke Tests Report
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      if: always() # always run even if the previous step fails
+      with:
+        name: surefire-test-results
+        path: '**/surefire-reports/TEST-*.xml'
+        retention-days: 1
+    - name: Cleanup Java Environment
+      uses: ./.github/actions/cleanup-java-env
+  java_integration_tests_templates:
+    name: Dataflow Templates Integration Tests
+    needs: [java_integration_smoke_tests_templates]
+    timeout-minutes: 180
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [self-hosted, it]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      with:
+        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Integration Tests
+      run: |
+        ./cicd/run-it-tests \
+        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
+        --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" \
+        --it-region="us-central1" \
+        --it-project="cloud-teleport-testing" \
+        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+        --it-private-connectivity="datastream-private-connect-us-central1"
+    - name: Upload Integration Tests Report
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      if: always() # always run even if the previous step fails
+      with:
+        name: surefire-test-results
+        path: '**/surefire-reports/TEST-*.xml'
+        retention-days: 1
+    - name: Cleanup Java Environment
+      uses: ./.github/actions/cleanup-java-env
+  java_load_tests_templates:
+    if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
+    name: Dataflow Templates Load Tests
+    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates]
+    timeout-minutes: 600
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [self-hosted, perf]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      with:
+        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Load Tests
+      run: |
+        ./cicd/run-load-tests \
+        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
+        --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" \
+        --it-region="us-central1" \
+        --it-project="cloud-teleport-testing" \
+        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+        --it-private-connectivity="datastream-private-connect-us-central1"
+    - name: Cleanup Java Environment
+      uses: ./.github/actions/cleanup-java-env

cicd/cmd/run-build/main.go

@@ -35,7 +35,8 @@ func main() {
 		mvnFlags.SkipShade(),
 		mvnFlags.SkipSpotlessCheck(),
 		mvnFlags.SkipTests(),
-		mvnFlags.ThreadCount(3))
+		mvnFlags.ThreadCount(3),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

cicd/cmd/run-checkstyle/main.go

@@ -28,7 +28,10 @@ func main() {
 	flags.RegisterCommonFlags()
 	flag.Parse()
 
-	if err := workflows.CheckstyleCheck().Run(); err != nil {
+	err := workflows.CheckstyleCheck().Run(
+		flags.ChangedModules())
+
+	if err != nil {
 		log.Fatalf("Error running checkstyle check: %v", err)
 	}
 	log.Println("Checkstyle check completed successfully!")

cicd/cmd/run-it-smoke-tests/main.go

@@ -41,7 +41,8 @@ func main() {
 		mvnFlags.SkipTests(),
 		mvnFlags.SkipJacoco(),
 		mvnFlags.SkipShade(),
-		mvnFlags.ThreadCount(8))
+		mvnFlags.ThreadCount(8),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}
@@ -65,7 +66,8 @@ func main() {
 		flags.StageBucket(),
 		flags.PrivateConnectivity(),
 		flags.FailureMode(),
-		flags.RetryFailures())
+		flags.RetryFailures(),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

cicd/cmd/run-it-tests/main.go

@@ -40,7 +40,8 @@ func main() {
 		mvnFlags.SkipTests(),
 		mvnFlags.SkipJacoco(),
 		mvnFlags.SkipShade(),
-		mvnFlags.ThreadCount(8))
+		mvnFlags.ThreadCount(8),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}
@@ -66,7 +67,8 @@ func main() {
 		flags.HostIp(),
 		flags.PrivateConnectivity(),
 		flags.FailureMode(),
-		flags.RetryFailures())
+		flags.RetryFailures(),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

cicd/cmd/run-load-tests/main.go

@@ -41,7 +41,8 @@ func main() {
 		mvnFlags.SkipTests(),
 		mvnFlags.SkipJacoco(),
 		mvnFlags.SkipShade(),
-		mvnFlags.ThreadCount(4))
+		mvnFlags.ThreadCount(4),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}
@@ -64,7 +65,8 @@ func main() {
 		flags.PrivateConnectivity(),
 		flags.ExportProject(),
 		flags.ExportDataset(),
-		flags.ExportTable())
+		flags.ExportTable(),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

cicd/cmd/run-spotless/main.go

@@ -28,7 +28,10 @@ func main() {
 	flags.RegisterCommonFlags()
 	flag.Parse()
 
-	if err := workflows.SpotlessCheck().Run(); err != nil {
+	err := workflows.SpotlessCheck().Run(
+		flags.ChangedModules())
+
+	if err != nil {
 		log.Fatalf("Error running spotless check: %v", err)
 	}
 	log.Println("Spotless check completed successfully!")

cicd/cmd/run-unit-tests/main.go

@@ -38,7 +38,8 @@ func main() {
 		mvnFlags.SkipSpotlessCheck(),
 		mvnFlags.SkipIntegrationTests(),
 		mvnFlags.FailAtTheEnd(),
-		mvnFlags.ThreadCount(8))
+		mvnFlags.ThreadCount(8),
+		flags.ChangedModules())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

cicd/internal/flags/common-flags.go

@@ -27,11 +27,13 @@ import (
 // Avoid making these vars public.
 var (
 	changedFiles string
+	changedModules string
 )
 
 // Registers all common flags. Must be called before flag.Parse().
 func RegisterCommonFlags() {
 	flag.StringVar(&changedFiles, "changed-files", "", "List of changed files as a comma-separated string")
+	flag.StringVar(&changedModules, "changed-modules", "", "List of changed modules as a comma-separated string")
 }
 
 // Returns all changed files with regexes. If no regexes are passed, all files are returned. If even one
@@ -68,3 +70,10 @@ func ChangedFiles(regexes ...string) []string {
 	}
 	return results
 }
+
+func ChangedModules() string {
+	if changedModules == "" {
+		return "ALL"
+	}
+	return "-pl=" + changedModules
+}

cicd/internal/flags/common-flags_test.go

@@ -97,3 +97,19 @@ func TestChangedFilesRegexesNoMatch(t *testing.T) {
 		t.Errorf("Expected empty slice but got %v", actual)
 	}
 }
+
+func TestChangedModulesEmpty(t *testing.T) {
+	changedModules = ""
+	actual := ChangedModules()
+	if actual != "" {
+		t.Errorf("Expected empty string but got %s", actual)
+	}
+}
+
+func TestChangedModulesNotEmpty(t *testing.T) {
+	changedModules = "v2/datastream-to-spanner"
+	actual := ChangedModules()
+	if actual != "-pl=v2/datastream-to-spanner" {
+		t.Errorf("Expected v2/datastream-to-spanner but got %s", actual)
+	}
+}