Update docker file

cloudbuild_CI.yaml

@@ -22,44 +22,21 @@ steps:
   - name: 'gcr.io/cloud-builders/docker'
     args:
       - 'build'
+      - '--build-arg'
+      - 'commit_sha=${COMMIT_SHA}'
       - '--tag=gcr.io/${PROJECT_ID}/gcp-variant-transforms:${COMMIT_SHA}'
       - '--file=docker/Dockerfile'
       - '.'
     id: 'build-gcp-variant-transforms-docker'
 
-  # We have to push now since we are using this image in the next step.
   - name: 'gcr.io/cloud-builders/docker'
     args:
       - 'push'
       - 'gcr.io/${PROJECT_ID}/gcp-variant-transforms:${COMMIT_SHA}'
     id: 'push-gcp-variant-transforms-docker'
 
-  - name: 'gcr.io/cloud-builders/docker'
-    args:
-      - 'build'
-      - '--tag=gcr.io/${PROJECT_ID}/gcp-variant-transforms-runner:${COMMIT_SHA}'
-      - '--file=docker/pipelines_runner/Dockerfile'
-      - '.'
-    id: 'build-gcp-variant-transforms-runner-docker'
-
-  - name: 'gcr.io/cloud-builders/docker'
-    args:
-      - 'push'
-      - 'gcr.io/${PROJECT_ID}/gcp-variant-transforms-runner:${COMMIT_SHA}'
-    id: 'push-gcp-variant-transforms-runner-docker'
-
-  # TODO(yifangchen): add the test for the image above.
-  # Run the test script from the first image we built and pushed.
-  #
-  # Note that ./deploy_and_run_tests.sh that is referenced here is coming from
-  # the source downloaded from GitHub and not the file in the docker image.
-  # The only reason that we use the built image in the 'name' argument is
-  # convenience, i.e., to avoid installing python, virtualenv, etc. on the
-  # image.
   - name: 'gcr.io/${PROJECT_ID}/gcp-variant-transforms:${COMMIT_SHA}'
     args:
-      - 'bash'
-      - './deploy_and_run_tests.sh'
       - '--keep_image'
       - '--skip_build'
       - '--project ${PROJECT_ID}'
@@ -70,6 +47,8 @@ steps:
       - '--run_all_tests'
       - '--test_name_prefix cloud-ci-'
     id: 'test-gcp-variant-transforms-docker'
+    entrypoint: '/opt/gcp_variant_transforms/src/deploy_and_run_tests.sh'
+
       # By default the script uses a GS bucket of gcp-variant-transforms-test
       # project. For other projects we should either use the following option
       # or change the script to create a temporary bucket on the fly.

docker/Dockerfile

@@ -17,6 +17,9 @@
 
 FROM google/cloud-sdk
 
+ARG commit_sha
+ENV COMMIT_SHA=${commit_sha}
+
 RUN mkdir -p /opt/gcp_variant_transforms/bin && mkdir -p /opt/gcp_variant_transforms/src
 ADD / /opt/gcp_variant_transforms/src/
 
@@ -47,3 +50,5 @@ RUN printf '#!/bin/bash\n%s\n%s' \
       'python -m gcp_variant_transforms.bq_to_vcf --setup_file ./setup.py "$@"' > \
       /opt/gcp_variant_transforms/bin/bq_to_vcf && \
     chmod +x /opt/gcp_variant_transforms/bin/bq_to_vcf
+
+ENTRYPOINT ["/opt/gcp_variant_transforms/src/docker/pipelines_runner.sh"]

docker/pipelines_runner/pipelines_runner.sh → docker/pipelines_runner.sh

@@ -54,7 +54,7 @@ function main {
   parse_args "$@"
 
   google_cloud_project="${google_cloud_project:-$(gcloud config get-value project)}"
-  vt_docker_image="${vt_docker_image:-gcr.io/gcp-variant-transforms/gcp-variant-transforms}"
+  vt_docker_image="${vt_docker_image:-gcr.io/gcp-variant-transforms/gcp-variant-transforms:${COMMIT_SHA}}"
   zones="${zones:-$(gcloud config get-value compute/zone)}"
   temp_location="${temp_location:-''}"
 
@@ -75,13 +75,16 @@ function main {
     exit 1
   fi
 
-  gcloud alpha genomics pipelines run \
+  operation_info=$( (`gcloud alpha genomics pipelines run \
     --project "${google_cloud_project}" \
     --logging "${temp_location}"/runner_logs_$(date +%Y%m%d_%H%M%S).log \
     --service-account-scopes "https://www.googleapis.com/auth/cloud-platform" \
     --zones "${zones}" \
     --docker-image "${vt_docker_image}" \
-    --command-line "/opt/gcp_variant_transforms/bin/${command} --project ${google_cloud_project}"
+    --command-line "/opt/gcp_variant_transforms/bin/${command} --project ${google_cloud_project}"`) 2>&1)
+
+  operation_id="$(echo ${operation_info} | grep -o -P '(?<=operations/).*(?=])')"
+  gcloud alpha genomics operations wait ${operation_id}
 }
 
 main "$@"

gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py

@@ -41,9 +41,8 @@
 from gcp_variant_transforms.testing.integration import run_tests_common
 
 
-_PIPELINE_NAME = 'gcp-variant-transforms-bq-to-vcf-integration-test'
 _TEST_FOLDER = 'gcp_variant_transforms/testing/integration/bq_to_vcf_tests'
-_SCRIPT_PATH = '/opt/gcp_variant_transforms/bin/bq_to_vcf'
+_TOOL_NAME = 'bq_to_vcf'
 
 
 class BqToVcfTestCase(run_tests_common.TestCaseInterface):
@@ -78,11 +77,11 @@ def __init__(self,
     for k, v in kwargs.iteritems():
       args.append('--{} {}'.format(k, v))
 
-    self.pipelines_api_request = run_tests_common.form_pipelines_api_request(
+    self.run_test_command = run_tests_common.form_command(
         parsed_args.project,
         filesystems.FileSystems.join(parsed_args.logging_location,
                                      '_'.join([test_name, timestamp])),
-        parsed_args.image, _PIPELINE_NAME, _SCRIPT_PATH, zones, args)
+        parsed_args.image, _TOOL_NAME, zones, args)
 
   def validate_result(self):
     """Validates the results.

gcp_variant_transforms/testing/integration/run_preprocessor_tests.py

@@ -44,8 +44,7 @@
 from gcp_variant_transforms.testing.integration import run_tests_common
 
 _BUCKET_NAME = 'integration_test_runs'
-_PIPELINE_NAME = 'gcp-variant-transforms-preprocessor-integration-test'
-_SCRIPT_PATH = '/opt/gcp_variant_transforms/bin/vcf_to_bq_preprocess'
+_TOOL_NAME = 'vcf_to_bq_preprocess'
 _TEST_FOLDER = 'gcp_variant_transforms/testing/integration/preprocessor_tests'
 
 
@@ -87,11 +86,11 @@ def __init__(self,
     for k, v in kwargs.iteritems():
       args.append('--{} {}'.format(k, v))
 
-    self.pipelines_api_request = run_tests_common.form_pipelines_api_request(
+    self.run_test_command = run_tests_common.form_command(
         parser_args.project,
         filesystems.FileSystems.join(parser_args.logging_location,
                                      self._report_blob_name),
-        parser_args.image, _PIPELINE_NAME, _SCRIPT_PATH, zones, args)
+        parser_args.image, _TOOL_NAME, zones, args)
 
   def validate_result(self):
     """Validates the results.

gcp_variant_transforms/testing/integration/run_tests_common.py

@@ -22,16 +22,14 @@
 import argparse  # pylint: disable=unused-import
 import json
 import os
+import subprocess
 import time
 from collections import namedtuple
 from typing import Dict, List, Optional  # pylint: disable=unused-import
 
-from googleapiclient import discovery
-from oauth2client.client import GoogleCredentials
 
 _DEFAULT_IMAGE_NAME = 'gcr.io/gcp-variant-transforms/gcp-variant-transforms'
 _DEFAULT_ZONES = ['us-east1-b']
-_CLOUD_PLATFORM_SCOPE = 'https://www.googleapis.com/auth/cloud-platform'
 
 # `TestCaseState` saves current running test and the remaining tests in the same
 # test script (.json).
@@ -64,13 +62,9 @@ def __init__(self, tests, revalidate=False):
       revalidate: If True, only run the result validation part of the tests.
     """
     self._tests = tests
-    self._service = discovery.build(
-        'genomics',
-        'v2alpha1',
-        credentials=GoogleCredentials.get_application_default())
     self._revalidate = revalidate
-    self._operation_names_to_test_states = {}  # type: Dict[str, TestCaseState]
-    self._operation_names_to_test_case_names = {}  # type: Dict[str, str]
+    self._test_names_to_test_states = {}  # type: Dict[str, TestCaseState]
+    self._test_names_to_processes = {}  # type: Dict[str, subprocess.Popen]
 
   def run(self):
     """Runs all tests."""
@@ -89,56 +83,40 @@ def _run_test(self, test_cases):
     """Runs the first test case in `test_cases`.
 
     The first test case and the remaining test cases form `TestCaseState` and
-    are added into `_operation_names_to_test_states` for future usage.
+    are added into `_test_names_to_test_states` for future usage.
     """
     if not test_cases:
       return
-    # The following pylint hint is needed because `pipelines` is a method that
-    # is dynamically added to the returned `service` object above. See
-    # `googleapiclient.discovery.Resource._set_service_methods`.
-    # pylint: disable=no-member
-    request = self._service.pipelines().run(
-        body=test_cases[0].pipelines_api_request)
-    operation_name = request.execute()['name']
-    self._operation_names_to_test_states.update(
-        {operation_name: TestCaseState(test_cases[0], test_cases[1:])})
-    self._operation_names_to_test_case_names.update(
-        {operation_name: test_cases[0]._name})
+    self._test_names_to_test_states.update({
+        test_cases[0].get_name(): TestCaseState(test_cases[0], test_cases[1:])})
+    self._test_names_to_processes.update(
+        {test_cases[0].get_name(): subprocess.Popen(
+            test_cases[0].run_test_command, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)})
 
   def _wait_for_all_operations_done(self):
     """Waits until all operations are done."""
-    # pylint: disable=no-member
-    operations = self._service.projects().operations()
-    while self._operation_names_to_test_states:
+    while self._test_names_to_processes:
       time.sleep(10)
-      running_operation_names = self._operation_names_to_test_states.keys()
-      for operation_name in running_operation_names:
-        request = operations.get(name=operation_name)
-        response = request.execute()
-        if response['done']:
-          self._handle_failure(operation_name, response)
-          test_case_state = self._operation_names_to_test_states.get(
-              operation_name)
-          del self._operation_names_to_test_states[operation_name]
+      running_test_names = self._test_names_to_processes.keys()
+      for test_name in running_test_names:
+        running_proc = self._test_names_to_processes.get(test_name)
+        return_code = running_proc.poll()
+        if return_code is not None:
+          test_case_state = self._test_names_to_test_states.get(test_name)
+          self._handle_failure(running_proc, test_case_state.running_test)
+          del self._test_names_to_processes[test_name]
           test_case_state.running_test.validate_result()
           self._run_test(test_case_state.remaining_tests)
 
-  def _handle_failure(self, operation_name, response):
+  def _handle_failure(self, proc, test_case):
     """Raises errors if test case failed."""
-    if 'error' in response:
-      if 'message' in response['error']:
-        raise TestCaseFailure(
-            'Test case {} (Operation {}) failed: {}'.format(
-                self._operation_names_to_test_case_names[operation_name],
-                operation_name,
-                response['error']['message']))
-      else:
-        # This case should never happen.
-        raise TestCaseFailure(
-            'Test case {} (Operation {}) failed: No traceback. '
-            'See logs for more information on error.'.format(
-                self._operation_names_to_test_case_names[operation_name],
-                operation_name))
+    if proc.returncode != 0:
+      stdout, stderr = proc.communicate()
+      raise TestCaseFailure('Test case {} failed. stdout: {}, stderr: {}, '
+                            'return code: {}.'.format(test_case.get_name(),
+                                                      stdout, stderr,
+                                                      proc.returncode))
 
   def print_results(self):
     """Prints results of test cases."""
@@ -148,43 +126,14 @@ def print_results(self):
     return 0
 
 
-def form_pipelines_api_request(project,  # type: str
-                               logging_location,  # type: str
-                               image,  # type: str
-                               pipeline_name,  # type: str
-                               script_path,  # type: str
-                               zones,  # type: Optional[List[str]]
-                               args  # type: List[str]
-                              ):
-  # type: (...) -> Dict
-  return {
-      'pipeline': {
-          'actions': [
-              {
-                  'imageUri': image,
-                  'commands': ['/bin/sh', '-c', ' '.join([script_path] + args)]
-              },
-              {
-                  'imageUri': 'gcr.io/cloud-genomics-pipelines/io',
-                  'commands': [
-                      'sh', '-c',
-                      'gsutil cp /google/logs/output %s' % logging_location
-                  ],
-                  'flags': ['ALWAYS_RUN'],
-              }
-          ],
-          'resources': {
-              'projectId': project,
-              'zones': zones or _DEFAULT_ZONES,
-              'virtualMachine': {
-                  'machineType': 'n1-standard-1',
-                  'serviceAccount': {'scopes': [_CLOUD_PLATFORM_SCOPE]},
-                  'bootDiskSizeGb': 100,
-                  'labels': {'pipeline_name': pipeline_name}
-              }
-          }
-      }
-  }
+def form_command(project, temp_location, image, tool_name, zones, args):
+  # type: (str, str, str, str, Optional[List[str]], List[str]) -> List[str]
+  return ['/opt/gcp_variant_transforms/src/docker/pipelines_runner.sh',
+          '--project', project,
+          '--docker_image', image,
+          '--temp_location', temp_location,
+          '--zones', str(' '.join(zones or _DEFAULT_ZONES)),
+          ' '.join([tool_name] + args)]
 
 
 def add_args(parser):

gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py

@@ -50,12 +50,10 @@
 # TODO(bashir2): Figure out why pylint can't find this.
 # pylint: disable=no-name-in-module,import-error
 from google.cloud import bigquery
-from oauth2client.client import GoogleCredentials
 
 from gcp_variant_transforms.testing.integration import run_tests_common
 
-_PIPELINE_NAME = 'gcp-variant-transforms-vcf-to-bq-integration-test'
-_SCRIPT_PATH = '/opt/gcp_variant_transforms/bin/vcf_to_bq'
+_TOOL_NAME = 'vcf_to_bq'
 _BASE_TEST_FOLDER = 'gcp_variant_transforms/testing/integration/vcf_to_bq_tests'
 
 
@@ -87,10 +85,10 @@ def __init__(self,
       if isinstance(v, basestring):
         value = v.format(TABLE_NAME=self._table_name)
       args.append('--{} {}'.format(k, value))
-    self.pipelines_api_request = run_tests_common.form_pipelines_api_request(
+    self.run_test_command = run_tests_common.form_command(
         context.project,
         filesystems.FileSystems.join(context.logging_location, output_table),
-        context.image, _PIPELINE_NAME, _SCRIPT_PATH, zones, args)
+        context.image, _TOOL_NAME, zones, args)
 
   def validate_result(self):
     """Runs queries against the output table and verifies results."""
@@ -186,7 +184,6 @@ def __init__(self, args):
     self.temp_location = args.temp_location
     self.logging_location = args.logging_location
     self.project = args.project
-    self.credentials = GoogleCredentials.get_application_default()
     self.image = args.image
     self._keep_tables = args.keep_tables
     self.revalidation_dataset_id = args.revalidation_dataset_id