Merge pull request #53 from GoogleCloudPlatform/dev

Merging dev into master for release.
GoogleCloudPlatform · May 4, 2018 · 059016e · 059016e
2 parents 1d60a47 + c6512f2
commit 059016e
Show file tree

Hide file tree

Showing 7 changed files with 129 additions and 19 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -0,0 +1,16 @@
+coverage:
+  precision: 2
+  round: down
+  range: "70...100"
+  status:
+    project:
+      default:
+        enabled: yes
+        threshold: 1%
+      patch:
+        enabled: yes
+        threshold: 1%
+comment:
+  layout: "header, diff, tree, changes"
+  behavior: default
+  require_changes: yes
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,32 @@
+env:
+  global:
+  - secure: mMIoTEgpKEhOKj5iMl+QkgLISGMMfKQJu4X3lEvBeSVWx6VrtrD4crl5xQrm/8JkDUo3VjUkSMKoTtN8YW27tx3bnpZIC8Q71omWXtO1K4J3nKfUBj1gS2QEpt1je+IXgR+ZvwrgDNe9YS0EfcjRT5or8/LgXcqCb1O+LfOc3EheTs86MMZzFdaawj7TGAmV9GyKPjZKe9Hb0L0TqeYlXbQY8RFntn31uW8qspXy3cnqXLgpqi09knDafDJwBO7F4fwUuzy8vKqzJIk9AaQ7ZRNRXc6XteiH39f3bqT2XYlu75bYTLIj/4vi3sDZZ8YJFPxK3Yi4N0LWY47njOVKCiHMTqsBGHoTfSYmVJutGIjLcCTK0WuEji0G+WjJc18BLkGru2vFn8XFhm2FSedMiR84QaEziiWFJ+APm4nfuwjS/U/E+5CBB0A7difOEqDXwgJDhRvE0a2rmBV2ZK9VIrBAFn3gWEtgA1n6sCLknN4olCiFaOAEPHXeTX9AbI8B5NIWq3TYoHIbj3x8BAfch+4UbkZvq6UzHa5gd940pcbXHTlDPHBb5DtjTp3oRiMrj8dg3to0guK/O6OeMnYQENnVR64WPPQWOtV0qjeIDh9d0T/gcFI9sgfkisfzSKR2ipZsnwGbLr2PsG3h4zQ3L5kot474VSLrtGEhAKC0ctw=
+  - secure: oPk75+zbZx6pCDlwqvMUErh+wc6jkpXeQ6oj16gARwCkCpQWt8/Oi945Zz01YwpReYKFLeMzk7EhliYkpJvnqqcfaJH/XqOoCbSNdsLLTi1Vw0wJBBetNjMyICFU958FDRw8bqVdqdAvShne2FzvIJFvTucVlg3aAaqszuGsWRzdxduXOxW6jIUIC01qq1eSN0NlS2d3PcSn4rmc/uzUQUOCOWfCOoe9nl+gkGIUZEDA4+OT/OOArV6ORmYDscgwVkMDL3gUU+2N7k7TsYtiEyJqJrp/orEk9B/U3qnt2gD1GREtmVIKJOM3od6hLSjx0D8F7aJHj0CdDyQb2dASsSoi6ODJl2HaFhf3FBzS4es7cddLKTZfrsW6Jyfa1NGzOklo8EJfgAMt/rfTLmTt5EPBbq04bkLpZTl6Ldv6xNp+UKHLNYK7u1qPE2k7LfN5+j9LrGjK5eSV8QlMDHnrmzeEaHSeOeLYN9NYoBfHtAA+LO/fHCjOY2sshF1lWyoGcQl0KJ9LVeMECBl2bk6F3HtZlhVqL7Ztbfd96dENLt75Dpb9JwIajdbEdrzxPvDyKTFT5+ozsb2AVGtSn4qM/G2PLzNza4wrPhngJYxYs/WsDKVKfay1RQLxwFiMMmYHp6I0qajiWtuhOYiuQrm6fkW/ByIsFAhduwpKA+aq8pg=
+  - secure: mfz/cDa9n7aWw3Vo0du2dwd3yddJIa75W1fn6fz4cNTCwN3Astf6AKBqvlM+NA/zUS0DyfjKY3q1nOxySKbfchG/ZcW0pYsKF+JK7qwc9EciDKsb8pAoMaJv6+ztJfc2hd1p88x8AAvTyoyvXZQvVgsUprQw74KAvUiGimU5hCPGn8ejK6pG7CnGLyT2Di2g8S+isrOvl4R1s4xWnWEI3xUoNBunZCK1hGu4A5WBr/jcbCCD6ah6E/irMYBY/17r8DmO7M5oB1RzXlwjyN/I2RsP6MtXKUBraw30gggKxOqSSiQ2AoWs6jcetSe9u47BCsuaqAVVjHApvhuJ6jOJlYAVDmqH3rAdCpUwe9I2AWSHK3R8m8iMJDzo85WIDT/ISlWiIvoFLsCVe3d2FOSc7xfhLjnbxbUo4xVljvvcS3T1kOdlDdN9VIeOKqsZ6b+65DOInIXg25XPj50yznbcav2yorT1YaQpwwAyMBx4xHFchB1GJaNmotdr845pfMrEq84pbwnTCNU6XxEQ3N+o8nDodv5eX7X+aFtJfNX94L9XpNZqUtTrdRc+oRn3u+QKf/uPUDW1Ac3jyrW7w136oj7U9tqrW4B4/cGN9RI0gkWpXTV45Nsr+oPO4h58oDiOTLvlEFkC8W5YCz+hNuqPSy4EtXXaN1Td8O6UTUqbNSo=
+  - secure: Yhl65uI3Utp2Bj9M0hW6HgclGUPs7rXXKcU/9cH3iu8cNjsZG6tLWGEd7wZgEvbhtLle7AIP2kq2k7MbtNf3w8GtFjD/62ocSoOwpbV8sBeTe+bCPZn1W5mOcKCwopqFM6fl5LX1lMIDee040+9omzUU5jSDsuHJzVzAmlgLjqX6/5zjHNDOFQj6PjkDdUraXXnRdxuJSwXAmU5fHiAZuBjHMZF+jFMA3yBovwPEas51Ty4Wr28b4GI0pGTfEcIW0fF6EOZprdMjqiGLeZkYiSgRn6QhLA6GVUADwT4WQJ3ZAQnCIfp/JOo+hRnY02sHdvdLFbprwPORcM+FdP7PzvOP7zgi/TKJ9UZIIwn39yITXVh4M+/GqQmZ8Hb4cgUG+iAjCrxbl5wQRviwJNjXhdKsx02SPpOboAILNUn5opJNUZ0s9iaEFEd8kTVDMX7QBM0i7+eSksoujdJZ/rZiXqQCnX8EPcbDLbRS1wMgRoJyC7Sdixt5AHLDZ5x5yndgl0RShs25ZMG9b1Gso8YnvRHspjuttTDO+gOFdbm7pyWwWULCc8pK5Q4RnlXeXAjh5zKZrwsPy/1hnS8YBw3UdWwgoK1BaU0dzGTkXSS5FFBKOaje+rOFKyga9IPqn9TlBXJ3OqCNQE8QXT9jHnHqc3yu0aik28oFR9fTU/wWi5A=
+  - secure: ppj07RXkflLzn/oXljfV9Ghdfa8OvU/kkKifYJx4f3y9f+KPJXwYS0wKWdQ6zhfY0EtMnU/UM1psj/TvfpWPLf+EeoDeyrpi5Dinov0yZ5Zc9M3xa2XnX+++BiZv69z1YBlu1PAvADSpHu8nss0FaM2a67dugd/J/lo4ahYKaJcc/nO64i3JYSHNwnpNhMl2mgGVo9/M5Mk/khXwqaejH/sDovsfx6REqw0JueuSYqgrRxQHdZSaLfYJKh0V5HdHf03vvckSeBTBqHnryYfxx/FyXOCfSQP9zFs6qtmllADJA1NzkYg+7ZlUozlBRBOYXQwAMp2vW3S8bFxuJ5lFAd7frtW+HJjP4hMiaMBfaMvO3wcgpX1fVWyrXaF/awdgGFZviZe9hUArBiLRRfrVEo7KJw5t2B2KO3heLZwGymi4n/bIs0dfFPUnxVxGLB3u7LLLenO2Z4hwJFybwQ7cVRwhB3Yv720RKWt3YsXDVUZivlIjBSqRSEjlrjdVf5M73R1j7Pid5eOPZwl2ju/5/ysQzCAHDP7zHRgivlySre1+HYpiW0SAg6yTUy8sMKgInIMEhyzLlt8cngRCk75LOpsfoTHnV7z3nvc86qcgplhFF9XTgot2a2OUfGeszLZef0dv1VNEKa4qBLjDaslAqRsUTUywxFFizER4gnqgfzU=
+  - secure: dfJrl3TZv0SZ/YFZen0bZwySUubWaaJ3gMkXAgIJxMfnT8akdhKpVyY/Xx8P9V0aLqQd3qZx7/vkkM7J8/PX+CXc9tZc2n/EY8XIfm/tDOakNteCqM6CwNLBSc2Z3yjRHifTuyFnQUwN1v12Oi6SD+KmtfBHq+O3PCWF4u//77vl3OK8q59YIE2ENzLkZcQfr9inFV0apwU8kIpZ6yTdTLZl77j0clGVxvMnIUpfTqYLtzaXr0sBwGN9klo42U8MvtgKV7/LB3JwE3et9h3/owr7t/5wTRjPj6biRwiTpAwX+Lo9fj+nf76xZMMcxrty9CCr+EaWn0L7w64fYxxf2fq/imD7A1AOAurkifWf9Gdh3ThzUjAjzSUKITpsVYqp55emvRbfFnp6aj3l0IpJVkgBNEU3IlnfK7lUa2DnaNsWhLj4KU1aIQxStniefWrgHLiZ4iAr5yjVaKJ8jXHOIOdK7Jr/SYkIPqJiYW+/zXgxLmCkKEw0Y6gUKWiwcLLzp2pnjcYFxUojs8NELZd+NJCWKjRbZBlq3Z0wo5rJygTPfWZuKBx/MaroAaS+vk5rgNCM1uOzklNPINj5EovObWOiFLxjeL0ktLUi/yo2yOxbvPosovCsF9DYyolwBBeB7phytP0g84cf0lIZcFA4wStAb92n5iJmEVRlIIl3n+A=
+  - GOOGLE_APPLICATION_CREDENTIALS="client-secrets.json"
 language: java
 sudo: false
 jdk:
- - openjdk8
-script:
-  - mvn clean test
+- openjdk8
+cache:
+  directories:
+  - "$HOME/.m2/repository"
+before_install:
+- openssl aes-256-cbc -K $encrypted_d25b186b6dd5_key -iv $encrypted_d25b186b6dd5_iv
+  -in client-secrets.json.enc -out client-secrets.json -d
+stages:
+- UNIT_TESTS
+- E2E_TESTS
+jobs:
+  include:
+  - stage: UNIT_TESTS
+    script: mvn -q clean test
+    script: mvn -q cobertura:cobertura
+  - stage: E2E_TESTS
+    script: "./travis/run_e2e_tests.sh --project=${E2E_TEST_PROJECT_NAME} --bucket=${E2E_TEST_BUCKET}
+      --database=${E2E_TEST_DATABASE}"
+after_success:
+- bash <(curl -s https://codecov.io/bash)
diff --git a/README.md b/README.md
@@ -1,4 +1,10 @@
 # pontem
+
+**branch: master** | **branch: dev**
+:------------ | :------------
+[![Build Status](https://travis-ci.org/GoogleCloudPlatform/pontem.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/pontem)|[![Build Status](https://travis-ci.org/GoogleCloudPlatform/pontem.svg?branch=dev)](https://travis-ci.org/GoogleCloudPlatform/pontem)
+[![codecov](https://codecov.io/gh/GoogleCloudPlatform/pontem/branch/master/graph/badge.svg)](https://codecov.io/gh/GoogleCloudPlatform/pontem)|[![codecov](https://codecov.io/gh/GoogleCloudPlatform/pontem/branch/dev/graph/badge.svg)](https://codecov.io/gh/GoogleCloudPlatform/pontem)
+
 ## Overview
 pontem is an open-source project that enable users of Cloud Spanner
 to perform four key operations:

diff --git a/client-secrets.json.enc b/client-secrets.json.enc
diff --git a/pom.xml b/pom.xml
@@ -34,6 +34,7 @@
              with the same version of GAX, which goes all the way back to 1.0.2 -->
         <google-cloud-storage.version>1.0.2</google-cloud-storage.version>
         <java.version>1.8</java.version>
+        <gax.version>1.1.0</gax.version>
     </properties>
 
     <repositories>
@@ -61,6 +62,19 @@
                     <target>${java.version}</target>
                 </configuration>
             </plugin>
+            <!-- Adding code coverage plugin -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>cobertura-maven-plugin</artifactId>
+                <version>2.7</version>
+                <configuration>
+                    <formats>
+                        <format>html</format>
+                        <format>xml</format>
+                   </formats>
+                   <check />
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
@@ -94,6 +108,14 @@
     </profiles>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-sdks-java-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-core</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-spanner</artifactId>
@@ -110,6 +132,51 @@
             <groupId>com.google.apis</groupId>
             <artifactId>google-api-services-dataflow</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.api-client</groupId>
+            <artifactId>google-api-client</artifactId>
+            <exclusions>
+                <exclusion>
+                  <groupId>com.google.guava</groupId>
+                  <artifactId>guava-jdk5</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.http-client</groupId>
+            <artifactId>google-http-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava-jdk5</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.http-client</groupId>
+            <artifactId>google-http-client-jackson2</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava-jdk5</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.api</groupId>
+            <artifactId>gax</artifactId>
+            <version>${gax.version}</version>
+        </dependency>
+
 
         <!-- Test Deps -->
         <dependency>

diff --git a/src/main/java/com/google/cloud/pontem/CloudSpannerDatabaseBackup.java b/src/main/java/com/google/cloud/pontem/CloudSpannerDatabaseBackup.java
@@ -242,7 +242,8 @@ public static String getOutputPath(String baseFolderPath) {
 
   public static final String LIST_ALL_TABLES_SQL_QUERY =
       "SELECT table_name, parent_table_name FROM information_schema.tables AS t WHERE t"
-          + ".table_catalog = '' and t.table_schema = '' ORDER BY parent_table_name DESC";
+          + ".table_catalog = '' and t.table_schema = '' ORDER BY"
+          + " parent_table_name, table_name DESC";
 
   /**
    * Fetch all the table names in a specific cloud spanner database.
@@ -351,8 +352,8 @@ public static String getSqlQueryForTablesToBackup(ImmutableList<String> tablesTo
   }
 
   /**
-   * Temporary workaround put in until https://github.com/apache/beam/pull/4946 is live.
-   * Gets the list of tables to backup.
+   * Temporary workaround put in until https://github.com/apache/beam/pull/4946 is live. Gets the
+   * list of tables to backup.
    */
   public static ImmutableList<String> getTableNamesBeingBackedUp(
       String projectId, String instance, String databaseId, String tableNamesQuery, Util util) {

diff --git a/run_all_tests.sh → travis/run_e2e_tests.sh b/run_all_tests.sh → travis/run_e2e_tests.sh
@@ -62,17 +62,11 @@ fi
 
 echo "FINISHED parsing flags."
 
-# Run Unit Tests
-echo "BEGIN running unit tests."
-mvn clean test || exit 1
-
-echo "FINISHED running unit tests."
-
 # Run End-To-End (E2E) Tests
 
 ## Setup
 echo "BEGIN running E2E setup."
-mvn compile exec:java \
+mvn -q compile exec:java \
    -Dexec.mainClass=com.google.cloud.pontem.EndToEndHelper \
    -Dexec.args="--projectId=${GCP_PROJECT} \
                 --gcsRootBackupFolderPath=gs://${GCP_BUCKET}/backup \
@@ -84,7 +78,7 @@ echo "FINISHED setup phase."
 
 ## Backup
 echo "BEGIN backup."
-mvn clean compile exec:java  -Dexec.mainClass=com.google.cloud.pontem.CloudSpannerDatabaseBackup  -Dexec.args="--runner=DataflowRunner \
+mvn -q clean compile exec:java  -Dexec.mainClass=com.google.cloud.pontem.CloudSpannerDatabaseBackup  -Dexec.args="--runner=DataflowRunner \
  --project=${GCP_PROJECT} \
  --gcpTempLocation=gs://${GCP_BUCKET}/tmp \
  --inputSpannerInstanceId=${DATABASE_INSTANCE} \
@@ -96,7 +90,7 @@ echo "FINISHED backup phase."
 
 ## Verify Backup
 echo "BEGIN verify backup."
-mvn compile exec:java \
+mvn -q compile exec:java \
    -Dexec.mainClass=com.google.cloud.pontem.EndToEndHelper \
    -Dexec.args="--projectId=${GCP_PROJECT} \
                 --gcsRootBackupFolderPath=gs://${GCP_BUCKET}/backup \
@@ -108,7 +102,7 @@ echo "FINISHED verify backup phase."
 
 ## Tear Down Database
 echo "BEGIN database teardown."
-mvn compile exec:java \
+mvn -q compile exec:java \
    -Dexec.mainClass=com.google.cloud.pontem.EndToEndHelper \
    -Dexec.args="--projectId=${GCP_PROJECT} \
                 --gcsRootBackupFolderPath=gs://${GCP_BUCKET}/multi-backup \
@@ -120,7 +114,7 @@ echo "FINISHED tear down database phase."
 
 ## Restore From Backup
 echo "BEGIN restore from backup."
-mvn clean compile exec:java  -Dexec.mainClass=com.google.cloud.pontem.CloudSpannerDatabaseRestore  -Dexec.args="--runner=DataflowRunner \
+mvn -q clean compile exec:java  -Dexec.mainClass=com.google.cloud.pontem.CloudSpannerDatabaseRestore  -Dexec.args="--runner=DataflowRunner \
  --project=${GCP_PROJECT} \
  --gcpTempLocation=gs://${GCP_BUCKET}/tmp \
  --outputSpannerInstanceId=${DATABASE_INSTANCE} \
@@ -132,7 +126,7 @@ echo "FINISHED restore from backup phase."
 
 ## Verify Database Restore
 echo "BEGIN database restore verify."
-mvn compile exec:java \
+mvn -q compile exec:java \
    -Dexec.mainClass=com.google.cloud.pontem.EndToEndHelper \
    -Dexec.args="--projectId=${GCP_PROJECT} \
                 --gcsRootBackupFolderPath=gs://${GCP_BUCKET}/multi-backup \
@@ -144,7 +138,7 @@ echo "FINISHED database restore verify."
 
 ## Tear Down
 echo "BEGIN final tear down."
-mvn compile exec:java \
+mvn -q compile exec:java \
    -Dexec.mainClass=com.google.cloud.pontem.EndToEndHelper \
    -Dexec.args="--projectId=${GCP_PROJECT} \
                 --gcsRootBackupFolderPath=gs://${GCP_BUCKET}/backup \