diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml index ec617ebae7..4759c82fda 100644 --- a/.github/workflows/java-pr.yml +++ b/.github/workflows/java-pr.yml @@ -27,6 +27,14 @@ on: # This will make it easier to verify action changes don't break anything. - '.github/actions/setup-env/*' - '.github/workflows/java-pr.yml' + # Exclude spanner paths from global run + - '!v2/datastream-to-spanner/**' + - '!v2/spanner-common/**' + - '!v2/spanner-change-streams-to-sharded-file-sink/**' + - '!v2/gcs-to-sourcedb/**' + - '!v2/spanner-migrations-sdk/**' + - '!v2/spanner-custom-shard/**' + - '!v2/sourcedb-to-spanner/**' schedule: - cron: "0 */12 * * *" workflow_dispatch: diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml new file mode 100644 index 0000000000..92cdb76eb3 --- /dev/null +++ b/.github/workflows/spanner-pr.yml @@ -0,0 +1,221 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checks that are intended to run on PRs containing Java code. + +name: Spanner PR + +on: + pull_request: + branches: + - 'main' + paths: + # Include spanner paths only + - 'v2/datastream-to-spanner/**' + - 'v2/spanner-common/**' + - 'v2/spanner-change-streams-to-sharded-file-sink/**' + - 'v2/gcs-to-sourcedb/**' + - 'v2/spanner-migrations-sdk/**' + - 'v2/spanner-custom-shard/**' + - 'v2/sourcedb-to-spanner/**' + schedule: + - cron: "0 */12 * * *" + workflow_dispatch: + +concurrency: + group: java-pr-${{ github.event.issue.number || github.run_id }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + +permissions: read-all + +jobs: + spotless_check: + name: Spotless + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + with: + fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Spotless + run: | + ./cicd/run-spotless \ + --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ + --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" + checkstyle_check: + name: Checkstyle + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + with: + fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Checkstyle + run: | + ./cicd/run-checkstyle \ + --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ + --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" + java_build: + name: Build + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + with: + fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: | + ./cicd/run-build \ + --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ + --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_unit_tests: + name: Unit Tests + needs: [java_build] + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + with: + fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Unit Tests + run: | + ./cicd/run-unit-tests \ + --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ + --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" + - name: Upload Unit Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: GoogleCloudPlatform/DataflowTemplates + files: 'target/site/jacoco-aggregate/jacoco.xml' + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_smoke_tests_templates: + name: Dataflow Templates Integration Smoke Tests + needs: [spotless_check, checkstyle_check, java_build, java_unit_tests] + timeout-minutes: 60 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + with: + fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Smoke Tests + run: | + ./cicd/run-it-smoke-tests \ + --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ + --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Smoke Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_tests_templates: + name: Dataflow Templates Integration Tests + needs: [java_integration_smoke_tests_templates] + timeout-minutes: 180 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + with: + fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ + --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Integration Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_load_tests_templates: + if: contains(github.event.pull_request.labels.*.name, 'run-load-tests') + name: Dataflow Templates Load Tests + needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates] + timeout-minutes: 600 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, perf] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + with: + fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Load Tests + run: | + ./cicd/run-load-tests \ + --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ + --changed-modules="v2/datastream-to-spanner/,v2/spanner-common/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/spanner-migrations-sdk/,v2/spanner-custom-shard/,v2/sourcedb-to-spanner/" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env diff --git a/cicd/cmd/run-build/main.go b/cicd/cmd/run-build/main.go index bc0812a1a9..9b83ef9ac3 100644 --- a/cicd/cmd/run-build/main.go +++ b/cicd/cmd/run-build/main.go @@ -35,7 +35,8 @@ func main() { mvnFlags.SkipShade(), mvnFlags.SkipSpotlessCheck(), mvnFlags.SkipTests(), - mvnFlags.ThreadCount(3)) + mvnFlags.ThreadCount(3), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/cmd/run-checkstyle/main.go b/cicd/cmd/run-checkstyle/main.go index 158d314f4c..75aa3623d1 100644 --- a/cicd/cmd/run-checkstyle/main.go +++ b/cicd/cmd/run-checkstyle/main.go @@ -28,7 +28,10 @@ func main() { flags.RegisterCommonFlags() flag.Parse() - if err := workflows.CheckstyleCheck().Run(); err != nil { + err := workflows.CheckstyleCheck().Run( + flags.ChangedModules()) + + if err != nil { log.Fatalf("Error running checkstyle check: %v", err) } log.Println("Checkstyle check completed successfully!") diff --git a/cicd/cmd/run-it-smoke-tests/main.go b/cicd/cmd/run-it-smoke-tests/main.go index bc8067377b..168d171cb8 100644 --- a/cicd/cmd/run-it-smoke-tests/main.go +++ b/cicd/cmd/run-it-smoke-tests/main.go @@ -41,7 +41,8 @@ func main() { mvnFlags.SkipTests(), mvnFlags.SkipJacoco(), mvnFlags.SkipShade(), - mvnFlags.ThreadCount(8)) + mvnFlags.ThreadCount(8), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } @@ -65,7 +66,8 @@ func main() { flags.StageBucket(), flags.PrivateConnectivity(), flags.FailureMode(), - flags.RetryFailures()) + flags.RetryFailures(), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index 1c7c4fa2aa..0debd33656 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -40,7 +40,8 @@ func main() { mvnFlags.SkipTests(), mvnFlags.SkipJacoco(), mvnFlags.SkipShade(), - mvnFlags.ThreadCount(8)) + mvnFlags.ThreadCount(8), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } @@ -66,7 +67,8 @@ func main() { flags.HostIp(), flags.PrivateConnectivity(), flags.FailureMode(), - flags.RetryFailures()) + flags.RetryFailures(), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/cmd/run-load-tests/main.go b/cicd/cmd/run-load-tests/main.go index 7b89819c1a..0ea2c05874 100644 --- a/cicd/cmd/run-load-tests/main.go +++ b/cicd/cmd/run-load-tests/main.go @@ -41,7 +41,8 @@ func main() { mvnFlags.SkipTests(), mvnFlags.SkipJacoco(), mvnFlags.SkipShade(), - mvnFlags.ThreadCount(4)) + mvnFlags.ThreadCount(4), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } @@ -64,7 +65,8 @@ func main() { flags.PrivateConnectivity(), flags.ExportProject(), flags.ExportDataset(), - flags.ExportTable()) + flags.ExportTable(), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/cmd/run-spotless/main.go b/cicd/cmd/run-spotless/main.go index b0c89a4f9d..a4404333d1 100644 --- a/cicd/cmd/run-spotless/main.go +++ b/cicd/cmd/run-spotless/main.go @@ -28,7 +28,10 @@ func main() { flags.RegisterCommonFlags() flag.Parse() - if err := workflows.SpotlessCheck().Run(); err != nil { + err := workflows.SpotlessCheck().Run( + flags.ChangedModules()) + + if err != nil { log.Fatalf("Error running spotless check: %v", err) } log.Println("Spotless check completed successfully!") diff --git a/cicd/cmd/run-unit-tests/main.go b/cicd/cmd/run-unit-tests/main.go index eb1f322871..3cc1bd12d1 100644 --- a/cicd/cmd/run-unit-tests/main.go +++ b/cicd/cmd/run-unit-tests/main.go @@ -38,7 +38,8 @@ func main() { mvnFlags.SkipSpotlessCheck(), mvnFlags.SkipIntegrationTests(), mvnFlags.FailAtTheEnd(), - mvnFlags.ThreadCount(8)) + mvnFlags.ThreadCount(8), + flags.ChangedModules()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/internal/flags/common-flags.go b/cicd/internal/flags/common-flags.go index faf965f00f..65b48b2e85 100644 --- a/cicd/internal/flags/common-flags.go +++ b/cicd/internal/flags/common-flags.go @@ -27,11 +27,13 @@ import ( // Avoid making these vars public. var ( changedFiles string + changedModules string ) // Registers all common flags. Must be called before flag.Parse(). func RegisterCommonFlags() { flag.StringVar(&changedFiles, "changed-files", "", "List of changed files as a comma-separated string") + flag.StringVar(&changedModules, "changed-modules", "", "List of changed modules as a comma-separated string") } // Returns all changed files with regexes. If no regexes are passed, all files are returned. If even one @@ -68,3 +70,10 @@ func ChangedFiles(regexes ...string) []string { } return results } + +func ChangedModules() string { + if changedModules == "" { + return "ALL" + } + return "-pl=" + changedModules +} diff --git a/cicd/internal/flags/common-flags_test.go b/cicd/internal/flags/common-flags_test.go index 0881b1bb21..5a08d2c7d2 100644 --- a/cicd/internal/flags/common-flags_test.go +++ b/cicd/internal/flags/common-flags_test.go @@ -97,3 +97,19 @@ func TestChangedFilesRegexesNoMatch(t *testing.T) { t.Errorf("Expected empty slice but got %v", actual) } } + +func TestChangedModulesEmpty(t *testing.T) { + changedModules = "" + actual := ChangedModules() + if actual != "" { + t.Errorf("Expected empty string but got %s", actual) + } +} + +func TestChangedModulesNotEmpty(t *testing.T) { + changedModules = "v2/datastream-to-spanner" + actual := ChangedModules() + if actual != "-pl=v2/datastream-to-spanner" { + t.Errorf("Expected v2/datastream-to-spanner but got %s", actual) + } +} \ No newline at end of file diff --git a/cicd/internal/op/maven.go b/cicd/internal/op/maven.go index 74cee339cf..9696447cc3 100644 --- a/cicd/internal/op/maven.go +++ b/cicd/internal/op/maven.go @@ -17,6 +17,7 @@ package op import ( + "slices" "strings" ) @@ -28,8 +29,14 @@ func RunMavenOnPom(pom string, cmd string, args ...string) error { fullArgs = append(fullArgs, strings.Split(cmd, " ")...) fullArgs = append(fullArgs, "-f", pom) fullArgs = append(fullArgs, "-e") + //filter out modules parameter if it is not specified. This has to be filtered out this + //way because we use variable length args, and returning "" for changed modules also means + //an empty value in the args. This empty value ends up creating an extra blank space + //in the final maven command constructed in RunCmdAndStreamOutput. + //Instead of trimming spaces, deleting the empty arg so that it is not considered in mvn command generation + //is less error prone. + args = slices.DeleteFunc(args, func(s string) bool { return s == "ALL"}) fullArgs = append(fullArgs, args...) - return RunCmdAndStreamOutput("mvn", fullArgs) } @@ -39,5 +46,6 @@ func RunMavenOnPom(pom string, cmd string, args ...string) error { func RunMavenOnModule(pom string, cmd string, module string, args ...string) error { // fullArgs := []string{"-pl", module} // fullArgs = append(fullArgs, args...) + args = slices.DeleteFunc(args, func(s string) bool { return s == "ALL"}) return RunMavenOnPom(pom, cmd, args...) } diff --git a/v2/datastream-to-spanner/pom.xml b/v2/datastream-to-spanner/pom.xml index 7a354d0520..1d5deea5ee 100644 --- a/v2/datastream-to-spanner/pom.xml +++ b/v2/datastream-to-spanner/pom.xml @@ -33,6 +33,11 @@ + + com.google.cloud.teleport.v2 + common + ${project.version} + com.google.cloud.teleport.v2 datastream-common