Fix our build

.travis.yml

@@ -2,9 +2,9 @@ language: python
 python:
   - "2.7"
 install:
-  - pip install --upgrade .
-  - pip install pylint
-  - pip install python-coveralls
+  - python -m pip install --upgrade .
+  - python -m pip install pylint
+  - python -m pip install python-coveralls
 script:
   - ./run_presubmit.sh
 after_success:

README.md

@@ -107,8 +107,8 @@ source. First install git, python, pip, and virtualenv:
 
 ```bash
 sudo apt-get install -y git python-pip python-dev build-essential
-sudo pip install --upgrade pip
-sudo pip install --upgrade virtualenv
+sudo python -m pip install --upgrade pip
+sudo python -m pip install --upgrade virtualenv
 ```
 
 Run virtualenv, clone the repo, and install pip packages:
@@ -118,7 +118,7 @@ virtualenv venv
 source venv/bin/activate
 git clone https://github.com/googlegenomics/gcp-variant-transforms.git
 cd gcp-variant-transforms
-pip install --upgrade .
+python -m pip install --upgrade .
 ```
 
 You may use the

deploy_and_run_tests.sh

@@ -35,6 +35,7 @@ GS_DIR_OPT="--gs_dir"
 KEEP_IMAGE_OPT="--keep_image"
 IMAGE_TAG_OPT="--image_tag"
 PROJECT_OPT="--project"
+REGION_OPT="--region"
 RUN_UNIT_TEST_OPT="--run_unit_tests"
 RUN_PREPROCESSOR_TEST_OPT="--run_preprocessor_tests"
 RUN_BQ_TO_VCF_TEST_OPT="--run_bq_to_vcf_tests"
@@ -44,6 +45,7 @@ keep_image=""
 skip_build=""
 gs_dir="integration_test_runs"  # default GS dir to store logs, etc.
 project="gcp-variant-transforms-test"  # default project to use
+region="us-central1"  # default region to use
 run_unit_tests=""  # By default do not run unit-tests.
 run_preprocessor_tests=""  # By default skip preprocessor integration tests.
 run_bq_to_vcf_tests=""  # By default skip bq to vcf integration tests.
@@ -65,7 +67,7 @@ color_print() {
 #################################################
 usage() {
   echo "Usage: $0  [${GS_DIR_OPT} gs_dir] [${KEEP_IMAGE_OPT}]"
-  echo "    [${IMAGE_TAG_OPT} image_tag] [${PROJECT_OPT} project_name] "
+  echo "    [${IMAGE_TAG_OPT} image_tag] [${PROJECT_OPT} project_name] [${REGION_OPT} region]"
   echo "    [${RUN_UNIT_TEST_OPT}] [${SKIP_BUILD_OPT}] [... test script options ...]"
   echo "  ${GS_DIR_OPT} can be used to change the default GS bucket used for"
   echo "    test run artifacts like logs, staging, etc. This has to be set if"
@@ -76,6 +78,7 @@ usage() {
   echo "  ${PROJECT_OPT} sets the cloud project. This project is used to push "
   echo "    the image, create BigQuery tables, run Genomics pipelines etc."
   echo "    Default is gcp-variant-transforms-test."
+  echo "  ${REGION_OPT} sets the region to run the BEAM pipelines"
   echo "  ${RUN_UNIT_TEST_OPT} runs the unit-tests before integration tests."
   echo "  ${RUN_PREPROCESSOR_TEST_OPT} runs the preprocessor integration tests."
   echo "  ${RUN_BQ_TO_VCF_TEST_OPT} runs the BQ to VCF integration tests."
@@ -148,6 +151,16 @@ parse_args() {
       project="$1"
       color_print "Using project: ${project}" "${GREEN}"
       shift
+    elif [[ "$1" = "${REGION_OPT}" ]]; then
+      shift
+      if [[ $# == 0 ]]; then
+        usage
+        color_print "ERROR: No name provided after ${REGION_OPT}!" "${RED}"
+        exit 1
+      fi
+      region="$1"
+      color_print "Using region: ${region}" "${GREEN}"
+      shift
     elif [[ "$1" = "${RUN_UNIT_TEST_OPT}" ]]; then
       run_unit_tests="yes"  # can be any non-empty string
       shift
@@ -222,17 +235,18 @@ virtualenv "${temp_dir}"
 source ${temp_dir}/bin/activate;
 trap clean_up EXIT
 if [[ -n "${run_unit_tests}" ]]; then
-  pip install --upgrade .
+  python -m pip install --upgrade .
   python setup.py test
 fi
-pip install --upgrade .[int_test]
+python -m pip install --upgrade .[int_test]
 
 # Force an upgrade to avoid SSL certificate verification errors (issue #453).
-pip install --upgrade httplib2
+python -m pip install --upgrade httplib2
 
 color_print "Running integration tests against ${full_image_name}" "${GREEN}"
 python gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py \
     --project "${project}" \
+    --region "${region}" \
     --staging_location "gs://${gs_dir}/staging" \
     --temp_location "gs://${gs_dir}/temp" \
     --logging_location "gs://${gs_dir}/temp/logs" \
@@ -241,6 +255,7 @@ pid_vcf_to_bq="$!"
 if [[ -n "${run_preprocessor_tests}" ]]; then
   python gcp_variant_transforms/testing/integration/run_preprocessor_tests.py \
       --project "${project}" \
+      --region "${region}" \
       --staging_location "gs://${gs_dir}/staging" \
       --temp_location "gs://${gs_dir}/temp" \
       --logging_location "gs://${gs_dir}/temp/logs" \
@@ -252,6 +267,7 @@ pid_preprocess="$!"
 if [[ -n "${run_bq_to_vcf_tests}" ]]; then
   python gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py \
       --project "${project}" \
+      --region "${region}" \
       --staging_location "gs://${gs_dir}/staging" \
       --temp_location "gs://${gs_dir}/temp" \
       --logging_location "gs://${gs_dir}/temp/logs" \

docker/Dockerfile

@@ -33,11 +33,12 @@ ADD / /opt/gcp_variant_transforms/src/
 RUN apt install -y g++
 
 # Install dependencies.
-RUN pip install --upgrade pip && pip install --upgrade virtualenv && \
+RUN python -m pip install --upgrade pip && \
+    python -m pip install --upgrade virtualenv && \
     virtualenv /opt/gcp_variant_transforms/venv && \
     . /opt/gcp_variant_transforms/venv/bin/activate && \
     cd /opt/gcp_variant_transforms/src && \
-    pip install --upgrade --no-binary pyvcf .
+    python -m pip install --upgrade --no-binary pyvcf .
 
 RUN printf '#!/bin/bash\n%s\n%s' \
       ". /opt/gcp_variant_transforms/venv/bin/activate && cd /opt/gcp_variant_transforms/src" \

docs/development_guide.md

@@ -43,16 +43,16 @@ git remote add upstream git@github.com:googlegenomics/gcp-variant-transforms.git
 
 ```bash
 sudo apt-get install python-pip python-dev build-essential
-sudo pip install --upgrade pip
-sudo pip install --upgrade virtualenv
+sudo python -m pip install --upgrade pip
+sudo python -m pip install --upgrade virtualenv
 virtualenv venv
 . venv/bin/activate
 ```
 
 #### Install dependences
 
 ```bash
-pip install --upgrade .
+python -m pip install --upgrade .
 ```
 Note that after running the above command we get some dependency conflicts in
 installed packages which is currently safe to ignore. For details see
@@ -155,7 +155,7 @@ script.
 Before pushing changes, make sure the pylint checks pass. To install pylint:
 ```bash
 source [PATH_TO_VENV]/bin/activate
-pip install --upgrade pylint
+python -m pip install --upgrade pylint
 ```
 Then run:
 ```bash

gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py

@@ -22,6 +22,7 @@
 Execute the following command from the root source directory:
 python gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py \
   --project gcp-variant-transforms-test \
+  --region us-central1 \
   --staging_location gs://integration_test_runs/staging \
   --temp_location gs://integration_test_runs/temp \
   --logging_location gs://integration_test_runs/temp/integration_test_logs
@@ -65,11 +66,10 @@ def __init__(self,
     self._output_file = filesystems.FileSystems.join(parsed_args.temp_location,
                                                      timestamp,
                                                      output_file_name)
-    self._project = parsed_args.project
-
     args = ['--input_table {}'.format(input_table),
             '--output_file {}'.format(self._output_file),
             '--project {}'.format(parsed_args.project),
+            '--region {}'.format(parsed_args.region),
             '--staging_location {}'.format(parsed_args.staging_location),
             '--temp_location {}'.format(parsed_args.temp_location),
             '--job_name {}'.format(

gcp_variant_transforms/testing/integration/run_preprocessor_tests.py

@@ -22,6 +22,7 @@
 Execute the following command from the root source directory:
 python gcp_variant_transforms/testing/integration/run_preprocessor_tests.py \
   --project gcp-variant-transforms-test \
+  --region us-central1 \
   --staging_location gs://integration_test_runs/staging \
   --temp_location gs://integration_test_runs/temp \
   --logging_location gs://integration_test_runs/temp/integration_test_logs
@@ -71,6 +72,7 @@ def __init__(self,
     self._project = parser_args.project
     args = ['--report_path {}'.format(self._report_path),
             '--project {}'.format(parser_args.project),
+            '--region {}'.format(parser_args.region),
             '--staging_location {}'.format(parser_args.staging_location),
             '--temp_location {}'.format(parser_args.temp_location),
             '--job_name {}'.format(

gcp_variant_transforms/testing/integration/run_tests_common.py

@@ -140,6 +140,7 @@ def add_args(parser):
   # type: (argparse.ArgumentParser) -> None
   """Adds common arguments."""
   parser.add_argument('--project', required=True)
+  parser.add_argument('--region', required=True)
   parser.add_argument('--staging_location', required=True)
   parser.add_argument('--temp_location', required=True)
   parser.add_argument('--logging_location', required=True)

gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py

@@ -24,6 +24,7 @@
 Execute the following command from the root source directory:
 python gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py \
   --project myproject \
+  --region us-central1 \
   --staging_location gs://mybucket/staging \
   --temp_location gs://mybucket/temp \
   --logging_location gs://mybucket/temp/integration_test_logs
@@ -77,6 +78,7 @@ def __init__(self,
     self._assertion_configs = assertion_configs
     args = ['--output_table {}'.format(output_table),
             '--project {}'.format(context.project),
+            '--region {}'.format(context.region),
             '--staging_location {}'.format(context.staging_location),
             '--temp_location {}'.format(context.temp_location),
             '--job_name {}-{}'.format(test_name, dataset_id.replace('_', '-'))]
@@ -184,6 +186,7 @@ def __init__(self, args):
     self.temp_location = args.temp_location
     self.logging_location = args.logging_location
     self.project = args.project
+    self.region = args.region
     self.image = args.image
     self._keep_tables = args.keep_tables
     self.revalidation_dataset_id = args.revalidation_dataset_id

run_presubmit.sh

@@ -20,13 +20,13 @@ set -euo pipefail
 echo ========== Running unit tests.
 if [[ -z `which coverage` ]];then
   echo "coverage is not installed. Installing ..."
-  pip install coverage
+  python -m pip install coverage
 fi
 coverage run --source=gcp_variant_transforms setup.py test
 
 echo ========== Running pylint.
 if [[ -z `which pylint` ]];then
   echo "pylint is not installed. Installing ..."
-  pip install pylint
+  python -m pip install pylint
 fi
-pylint gcp_variant_transforms
+python -m pylint gcp_variant_transforms

setup.py

@@ -29,11 +29,7 @@
     # Nucleus needs uptodate protocol buffer compiler (protoc).
     'protobuf>=3.6.1',
     'mmh3<2.6',
-    # Need to explicitly install v<=1.14.0. apache-beam requires
-    # google-cloud-pubsub 0.39.1, which relies on google-cloud-core<0.30dev,
-    # >=0.29.1. google-cloud-storage also has requirements on google-cloud-core,
-    # and version 1.14.0 resolves the dependency conflicts.
-    'google-cloud-storage<=1.14.0'
+    'google-cloud-storage'
 ]
 
 REQUIRED_SETUP_PACKAGES = [