diff --git a/.travis.yml b/.travis.yml index 97dd3d55a..2e6afd640 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,9 @@ language: python python: - "2.7" install: - - pip install --upgrade . - - pip install pylint - - pip install python-coveralls + - python -m pip install --upgrade . + - python -m pip install pylint + - python -m pip install python-coveralls script: - ./run_presubmit.sh after_success: diff --git a/README.md b/README.md index 97aeb8dd7..127532f65 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,8 @@ source. First install git, python, pip, and virtualenv: ```bash sudo apt-get install -y git python-pip python-dev build-essential -sudo pip install --upgrade pip -sudo pip install --upgrade virtualenv +sudo python -m pip install --upgrade pip +sudo python -m pip install --upgrade virtualenv ``` Run virtualenv, clone the repo, and install pip packages: @@ -118,7 +118,7 @@ virtualenv venv source venv/bin/activate git clone https://github.com/googlegenomics/gcp-variant-transforms.git cd gcp-variant-transforms -pip install --upgrade . +python -m pip install --upgrade . ``` You may use the diff --git a/deploy_and_run_tests.sh b/deploy_and_run_tests.sh index 872652f0c..f8efb165e 100755 --- a/deploy_and_run_tests.sh +++ b/deploy_and_run_tests.sh @@ -35,6 +35,7 @@ GS_DIR_OPT="--gs_dir" KEEP_IMAGE_OPT="--keep_image" IMAGE_TAG_OPT="--image_tag" PROJECT_OPT="--project" +REGION_OPT="--region" RUN_UNIT_TEST_OPT="--run_unit_tests" RUN_PREPROCESSOR_TEST_OPT="--run_preprocessor_tests" RUN_BQ_TO_VCF_TEST_OPT="--run_bq_to_vcf_tests" @@ -44,6 +45,7 @@ keep_image="" skip_build="" gs_dir="integration_test_runs" # default GS dir to store logs, etc. project="gcp-variant-transforms-test" # default project to use +region="us-central1" # default region to use run_unit_tests="" # By default do not run unit-tests. run_preprocessor_tests="" # By default skip preprocessor integration tests. run_bq_to_vcf_tests="" # By default skip bq to vcf integration tests. @@ -65,7 +67,7 @@ color_print() { ################################################# usage() { echo "Usage: $0 [${GS_DIR_OPT} gs_dir] [${KEEP_IMAGE_OPT}]" - echo " [${IMAGE_TAG_OPT} image_tag] [${PROJECT_OPT} project_name] " + echo " [${IMAGE_TAG_OPT} image_tag] [${PROJECT_OPT} project_name] [${REGION_OPT} region]" echo " [${RUN_UNIT_TEST_OPT}] [${SKIP_BUILD_OPT}] [... test script options ...]" echo " ${GS_DIR_OPT} can be used to change the default GS bucket used for" echo " test run artifacts like logs, staging, etc. This has to be set if" @@ -76,6 +78,7 @@ usage() { echo " ${PROJECT_OPT} sets the cloud project. This project is used to push " echo " the image, create BigQuery tables, run Genomics pipelines etc." echo " Default is gcp-variant-transforms-test." + echo " ${REGION_OPT} sets the region to run the BEAM pipelines" echo " ${RUN_UNIT_TEST_OPT} runs the unit-tests before integration tests." echo " ${RUN_PREPROCESSOR_TEST_OPT} runs the preprocessor integration tests." echo " ${RUN_BQ_TO_VCF_TEST_OPT} runs the BQ to VCF integration tests." @@ -148,6 +151,16 @@ parse_args() { project="$1" color_print "Using project: ${project}" "${GREEN}" shift + elif [[ "$1" = "${REGION_OPT}" ]]; then + shift + if [[ $# == 0 ]]; then + usage + color_print "ERROR: No name provided after ${REGION_OPT}!" "${RED}" + exit 1 + fi + region="$1" + color_print "Using region: ${region}" "${GREEN}" + shift elif [[ "$1" = "${RUN_UNIT_TEST_OPT}" ]]; then run_unit_tests="yes" # can be any non-empty string shift @@ -222,17 +235,18 @@ virtualenv "${temp_dir}" source ${temp_dir}/bin/activate; trap clean_up EXIT if [[ -n "${run_unit_tests}" ]]; then - pip install --upgrade . + python -m pip install --upgrade . python setup.py test fi -pip install --upgrade .[int_test] +python -m pip install --upgrade .[int_test] # Force an upgrade to avoid SSL certificate verification errors (issue #453). -pip install --upgrade httplib2 +python -m pip install --upgrade httplib2 color_print "Running integration tests against ${full_image_name}" "${GREEN}" python gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py \ --project "${project}" \ + --region "${region}" \ --staging_location "gs://${gs_dir}/staging" \ --temp_location "gs://${gs_dir}/temp" \ --logging_location "gs://${gs_dir}/temp/logs" \ @@ -241,6 +255,7 @@ pid_vcf_to_bq="$!" if [[ -n "${run_preprocessor_tests}" ]]; then python gcp_variant_transforms/testing/integration/run_preprocessor_tests.py \ --project "${project}" \ + --region "${region}" \ --staging_location "gs://${gs_dir}/staging" \ --temp_location "gs://${gs_dir}/temp" \ --logging_location "gs://${gs_dir}/temp/logs" \ @@ -252,6 +267,7 @@ pid_preprocess="$!" if [[ -n "${run_bq_to_vcf_tests}" ]]; then python gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py \ --project "${project}" \ + --region "${region}" \ --staging_location "gs://${gs_dir}/staging" \ --temp_location "gs://${gs_dir}/temp" \ --logging_location "gs://${gs_dir}/temp/logs" \ diff --git a/docker/Dockerfile b/docker/Dockerfile index 8add9d4b5..373d7e208 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,11 +33,12 @@ ADD / /opt/gcp_variant_transforms/src/ RUN apt install -y g++ # Install dependencies. -RUN pip install --upgrade pip && pip install --upgrade virtualenv && \ +RUN python -m pip install --upgrade pip && \ + python -m pip install --upgrade virtualenv && \ virtualenv /opt/gcp_variant_transforms/venv && \ . /opt/gcp_variant_transforms/venv/bin/activate && \ cd /opt/gcp_variant_transforms/src && \ - pip install --upgrade --no-binary pyvcf . + python -m pip install --upgrade --no-binary pyvcf . RUN printf '#!/bin/bash\n%s\n%s' \ ". /opt/gcp_variant_transforms/venv/bin/activate && cd /opt/gcp_variant_transforms/src" \ diff --git a/docs/development_guide.md b/docs/development_guide.md index 807f64173..33d5ecdc7 100644 --- a/docs/development_guide.md +++ b/docs/development_guide.md @@ -43,8 +43,8 @@ git remote add upstream git@github.com:googlegenomics/gcp-variant-transforms.git ```bash sudo apt-get install python-pip python-dev build-essential -sudo pip install --upgrade pip -sudo pip install --upgrade virtualenv +sudo python -m pip install --upgrade pip +sudo python -m pip install --upgrade virtualenv virtualenv venv . venv/bin/activate ``` @@ -52,7 +52,7 @@ virtualenv venv #### Install dependences ```bash -pip install --upgrade . +python -m pip install --upgrade . ``` Note that after running the above command we get some dependency conflicts in installed packages which is currently safe to ignore. For details see @@ -155,7 +155,7 @@ script. Before pushing changes, make sure the pylint checks pass. To install pylint: ```bash source [PATH_TO_VENV]/bin/activate -pip install --upgrade pylint +python -m pip install --upgrade pylint ``` Then run: ```bash diff --git a/gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py b/gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py index e1768dd0c..2bea451cf 100644 --- a/gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py +++ b/gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py @@ -22,6 +22,7 @@ Execute the following command from the root source directory: python gcp_variant_transforms/testing/integration/run_bq_to_vcf_tests.py \ --project gcp-variant-transforms-test \ + --region us-central1 \ --staging_location gs://integration_test_runs/staging \ --temp_location gs://integration_test_runs/temp \ --logging_location gs://integration_test_runs/temp/integration_test_logs @@ -65,11 +66,10 @@ def __init__(self, self._output_file = filesystems.FileSystems.join(parsed_args.temp_location, timestamp, output_file_name) - self._project = parsed_args.project - args = ['--input_table {}'.format(input_table), '--output_file {}'.format(self._output_file), '--project {}'.format(parsed_args.project), + '--region {}'.format(parsed_args.region), '--staging_location {}'.format(parsed_args.staging_location), '--temp_location {}'.format(parsed_args.temp_location), '--job_name {}'.format( diff --git a/gcp_variant_transforms/testing/integration/run_preprocessor_tests.py b/gcp_variant_transforms/testing/integration/run_preprocessor_tests.py index 04f9e2608..c079526f4 100644 --- a/gcp_variant_transforms/testing/integration/run_preprocessor_tests.py +++ b/gcp_variant_transforms/testing/integration/run_preprocessor_tests.py @@ -22,6 +22,7 @@ Execute the following command from the root source directory: python gcp_variant_transforms/testing/integration/run_preprocessor_tests.py \ --project gcp-variant-transforms-test \ + --region us-central1 \ --staging_location gs://integration_test_runs/staging \ --temp_location gs://integration_test_runs/temp \ --logging_location gs://integration_test_runs/temp/integration_test_logs @@ -71,6 +72,7 @@ def __init__(self, self._project = parser_args.project args = ['--report_path {}'.format(self._report_path), '--project {}'.format(parser_args.project), + '--region {}'.format(parser_args.region), '--staging_location {}'.format(parser_args.staging_location), '--temp_location {}'.format(parser_args.temp_location), '--job_name {}'.format( diff --git a/gcp_variant_transforms/testing/integration/run_tests_common.py b/gcp_variant_transforms/testing/integration/run_tests_common.py index 4fe4bb677..daf36bd51 100644 --- a/gcp_variant_transforms/testing/integration/run_tests_common.py +++ b/gcp_variant_transforms/testing/integration/run_tests_common.py @@ -140,6 +140,7 @@ def add_args(parser): # type: (argparse.ArgumentParser) -> None """Adds common arguments.""" parser.add_argument('--project', required=True) + parser.add_argument('--region', required=True) parser.add_argument('--staging_location', required=True) parser.add_argument('--temp_location', required=True) parser.add_argument('--logging_location', required=True) diff --git a/gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py b/gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py index e0e9cb7c1..332efbf6a 100644 --- a/gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py +++ b/gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py @@ -24,6 +24,7 @@ Execute the following command from the root source directory: python gcp_variant_transforms/testing/integration/run_vcf_to_bq_tests.py \ --project myproject \ + --region us-central1 \ --staging_location gs://mybucket/staging \ --temp_location gs://mybucket/temp \ --logging_location gs://mybucket/temp/integration_test_logs @@ -77,6 +78,7 @@ def __init__(self, self._assertion_configs = assertion_configs args = ['--output_table {}'.format(output_table), '--project {}'.format(context.project), + '--region {}'.format(context.region), '--staging_location {}'.format(context.staging_location), '--temp_location {}'.format(context.temp_location), '--job_name {}-{}'.format(test_name, dataset_id.replace('_', '-'))] @@ -184,6 +186,7 @@ def __init__(self, args): self.temp_location = args.temp_location self.logging_location = args.logging_location self.project = args.project + self.region = args.region self.image = args.image self._keep_tables = args.keep_tables self.revalidation_dataset_id = args.revalidation_dataset_id diff --git a/run_presubmit.sh b/run_presubmit.sh index 0c4b17ae1..24eab63db 100755 --- a/run_presubmit.sh +++ b/run_presubmit.sh @@ -20,13 +20,13 @@ set -euo pipefail echo ========== Running unit tests. if [[ -z `which coverage` ]];then echo "coverage is not installed. Installing ..." - pip install coverage + python -m pip install coverage fi coverage run --source=gcp_variant_transforms setup.py test echo ========== Running pylint. if [[ -z `which pylint` ]];then echo "pylint is not installed. Installing ..." - pip install pylint + python -m pip install pylint fi -pylint gcp_variant_transforms +python -m pylint gcp_variant_transforms diff --git a/setup.py b/setup.py index fc0d6cc31..1fb6df435 100644 --- a/setup.py +++ b/setup.py @@ -29,11 +29,7 @@ # Nucleus needs uptodate protocol buffer compiler (protoc). 'protobuf>=3.6.1', 'mmh3<2.6', - # Need to explicitly install v<=1.14.0. apache-beam requires - # google-cloud-pubsub 0.39.1, which relies on google-cloud-core<0.30dev, - # >=0.29.1. google-cloud-storage also has requirements on google-cloud-core, - # and version 1.14.0 resolves the dependency conflicts. - 'google-cloud-storage<=1.14.0' + 'google-cloud-storage' ] REQUIRED_SETUP_PACKAGES = [