build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

plugins { id 'org.apache.beam.module' }
applyGoNature()
applyPythonNature()

description = "Apache Beam :: SDKs :: Go :: Test"

// ValidatesRunner tests for Dataflow. Runs tests in the integration directory
// with Dataflow to validate that the runner behaves as expected.
task dataflowValidatesRunner() {
  group = "Verification"

  dependsOn ":sdks:go:test:goBuild"
  dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"

  doLast {
    def pipelineOptions = [  // Pipeline options piped directly to Go SDK flags.
        "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}",
    ]
    def options = [
        "--runner dataflow",
        "--pipeline_opts \"${pipelineOptions.join(' ')}\"",
    ]
    exec {
      executable "sh"
      args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
    }
  }
}

// ValidatesRunner tests for Flink. Runs tests in the integration directory
// with Flink to validate that the runner behaves as expected.
task flinkValidatesRunner {
  group = "Verification"

  dependsOn ":sdks:go:test:goBuild"
  dependsOn ":sdks:go:container:docker"
  dependsOn ":sdks:java:container:java8:docker"
  dependsOn ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar"
  dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
  doLast {
    def pipelineOptions = [  // Pipeline options piped directly to Go SDK flags.
        "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}",
    ]
    def options = [
        "--runner flink",
        "--flink_job_server_jar ${project(":runners:flink:${project.ext.latestFlinkVersion}:job-server").shadowJar.archivePath}",
        "--pipeline_opts \"${pipelineOptions.join(' ')}\"",
    ]
    exec {
      executable "sh"
      args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
    }
  }
}

// ValidatesRunner tests for Samza. Runs tests in the integration directory
// with Samza to validate that the runner behaves as expected.
task samzaValidatesRunner {
  dependsOn ":sdks:go:test:goBuild"
  dependsOn ":sdks:go:container:docker"
  dependsOn ":sdks:java:container:java8:docker"
  dependsOn ":runners:samza:job-server:shadowJar"
  dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
  doLast {
    def pipelineOptions = [  // Pipeline options piped directly to Go SDK flags.
        "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}",
    ]
    def options = [
        "--runner samza",
        "--samza_job_server_jar ${project(":runners:samza:job-server").shadowJar.archivePath}",
        "--pipeline_opts \"${pipelineOptions.join(' ')}\"",
    ]
    exec {
      executable "sh"
      args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
    }
  }
}

// ValidatesRunner tests for Spark. Runs tests in the integration directory
// with Spark to validate that the runner behaves as expected.
task sparkValidatesRunner {
  group = "Verification"

  dependsOn ":sdks:go:test:goBuild"
  dependsOn ":sdks:java:container:java8:docker"
  dependsOn ":runners:spark:3:job-server:shadowJar"
  dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
  doLast {
    def pipelineOptions = [  // Pipeline options piped directly to Go SDK flags.
        "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}",
    ]
    def options = [
        "--runner spark",
        "--spark_job_server_jar ${project(":runners:spark:3:job-server").shadowJar.archivePath}",
        "--pipeline_opts \"${pipelineOptions.join(' ')}\"",
    ]
    exec {
      executable "sh"
      args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
    }
  }
}

// ValidatesRunner tests for the Python Portable runner (aka. ULR). Runs tests
// in the integration directory with the ULR to validate that the runner behaves
// as expected.
//
// The ULR can exhibit strange behavior when containers are built with outdated
// vendored directories. For best results use the clean task, like so:
//   ./gradlew clean :sdks:go:test:ulrValidatesRunner
tasks.register("ulrValidatesRunner") {
  group = "Verification"

  dependsOn ":sdks:go:test:goBuild"
  dependsOn ":sdks:go:container:docker"
  dependsOn ":sdks:java:container:java8:docker"
  dependsOn "setupVirtualenv"
  dependsOn ":sdks:python:buildPython"
  dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
  doLast {
    def pipelineOptions = [  // Pipeline options piped directly to Go SDK flags.
         "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}",
    ]
    def options = [
        "--runner portable",
        "--pipeline_opts \"${pipelineOptions.join(' ')}\"",
    ]
    exec {
      executable "sh"
      workingDir "${project.rootDir}/sdks/python"
      args "-c", ". ${envdir}/bin/activate && pip install -e ."
    }
    exec {
      executable "sh"
      args "-c", ". ${envdir}/bin/activate && ./run_validatesrunner_tests.sh ${options.join(' ')}"
    }
  }
}

// A method for configuring a cross-language validates runner test task,
// intended to be used in calls to createCrossLanguageValidatesRunnerTask.
ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts ->
  return proj.tasks.register(name) {
    group = "Verification"

    dependsOn ":sdks:go:test:goBuild"
    dependsOn ":sdks:go:container:docker"
    dependsOn ":sdks:java:io:expansion-service:build"
    dependsOn ":sdks:java:extensions:schemaio-expansion-service:build"
    dependsOn ":sdks:java:io:debezium:expansion-service:build"
    dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build"
    dependsOn ":sdks:java:testing:kafka-service:buildTestKafkaServiceJar"

    doLast {
      def ioExpJar = project(":sdks:java:io:expansion-service").shadowJar.archivePath
      def schemaIoExpJar = project(":sdks:java:extensions:schemaio-expansion-service").shadowJar.archivePath
      def debeziumIoExpJar = project(":sdks:java:io:debezium:expansion-service").shadowJar.archivePath
      def gcpIoExpJar = project(":sdks:java:io:google-cloud-platform:expansion-service").shadowJar.archivePath
      def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archivePath
      def pipelineOptions = [  // Pipeline options piped directly to Go SDK flags.
          "--kafka_jar=${kafkaJar}",
          "--expansion_jar=io:${ioExpJar}",
          "--expansion_jar=schemaio:${schemaIoExpJar}",
          "--expansion_jar=debeziumio:${debeziumIoExpJar}",
          "--expansion_jar=gcpio:${gcpIoExpJar}",
          "--bq_dataset=apache-beam-testing.beam_bigquery_io_test_temp",
      ]
      pipelineOptions.addAll(pipelineOpts)
      def options = [
          "--pipeline_opts \"${pipelineOptions.join(' ')}\"",
      ]
      options.addAll(scriptOpts)
      logger.info("Running the command: sh -c ./run_validatesrunner_tests.sh ${options.join(' ')}")
      exec {
        executable "sh"
        args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
      }
    }
  }
}