Skip to content
Permalink
Browse files
HOP-2932 : Add Big Table support to the Beam engines
HOP-2939 : Update Apache Beam API to 2.29.0 + jar version alignments
  • Loading branch information
mattcasters committed Jun 4, 2021
1 parent 7878935 commit ef114f00a93781abf3b9052ce5cf179a4adec23d
Showing 52 changed files with 3,529 additions and 2,217 deletions.
@@ -45,7 +45,7 @@
<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<version>1.30.10</version>
<version>${google-api-client.version}</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
@@ -50,10 +50,13 @@
</repositories>

<properties>
<apache-beam-version>2.28.0</apache-beam-version>
<apache-beam-version>2.29.0</apache-beam-version>
<spark-version>2.4.7</spark-version>
<flink-version>1.9.3</flink-version>
<flink-version>1.10.1</flink-version>
<json-simple.version>1.1.1</json-simple.version>
<bigtable-hbase-beam.version>1.20.1</bigtable-hbase-beam.version>
<grpc.version>1.37.0</grpc.version>
<gax-grpc.version>1.63.0</gax-grpc.version>
</properties>

<dependencies>
@@ -63,13 +66,75 @@
<version>0.99-SNAPSHOT</version>
</dependency>

<!-- Force the right BQ lib -->
<!-- Force the right libs version -->
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-bigquery</artifactId>
<version>v2-rev20191211-1.30.9</version>
</dependency>

<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf-lite</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-api</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-context</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-grpclb</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-auth</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-alts</artifactId>
<version>${grpc.version}</version>
</dependency>

<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
<version>${gax-grpc.version}</version>
</dependency>

<!-- The Apache Beam dependencies -->

<dependency>
@@ -160,7 +225,7 @@
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-flink-1.9</artifactId>
<artifactId>beam-runners-flink-1.10</artifactId>
<version>${apache-beam-version}</version>
<exclusions>
<exclusion>
@@ -371,6 +436,12 @@
<version>3.2.2</version>
</dependency>

<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-beam</artifactId>
<version>${bigtable-hbase-beam.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-sort</artifactId>
@@ -13,7 +13,6 @@
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->

<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
@@ -78,6 +77,7 @@
<include>com.fasterxml.jackson.core:jackson-databind</include>
<include>com.fasterxml.jackson.module:jackson-module-paranamer</include>
<include>com.fasterxml.jackson.module:jackson-module-scala_2.11</include>
<include>com.github.stephenc.findbugs:findbugs-annotations</include>
<include>com.github.luben:zstd-jni</include>
<include>com.github.scopt:scopt_2.11</include>
<include>com.google.android:annotations</include>
@@ -117,6 +117,7 @@
<include>com.google.cloud.bigdataoss:gcsio</include>
<include>com.google.cloud.bigdataoss:util</include>
<include>com.google.cloud.bigtable:bigtable-client-core</include>
<include>com.google.cloud.bigtable:bigtable-hbase-1.x-shaded</include>
<include>com.google.cloud.datastore:datastore-v1-proto-client</include>
<include>com.google.cloud:google-cloud-bigquery</include>
<include>com.google.cloud:google-cloud-bigquerystorage</include>
@@ -137,6 +138,7 @@
<include>com.google.http-client:google-http-client-appengine</include>
<include>com.google.http-client:google-http-client-jackson2</include>
<include>com.google.http-client:google-http-client-protobuf</include>
<include>com.google.http-client:google-http-client-gson</include>
<include>com.google.j2objc:j2objc-annotations</include>
<include>com.google.oauth-client:google-oauth-client</include>
<include>com.google.oauth-client:google-oauth-client-java6</include>
@@ -170,7 +172,9 @@
<include>io.dropwizard.metrics:metrics-json</include>
<include>io.dropwizard.metrics:metrics-jvm</include>
<include>io.github.classgraph:classgraph</include>
<include>io.grpc:grpc-all</include>

<!--<include>io.grpc:grpc-all</include>-->

<include>io.grpc:grpc-alts</include>
<include>io.grpc:grpc-api</include>
<include>io.grpc:grpc-auth</include>
@@ -184,8 +188,10 @@
<include>io.grpc:grpc-protobuf-lite</include>
<include>io.grpc:grpc-stub</include>
<include>io.grpc:grpc-testing</include>

<!--<include>io.netty:netty-all</include>-->

<include>io.netty:netty</include>
<include>io.netty:netty-all</include>
<include>io.netty:netty-buffer</include>
<include>io.netty:netty-codec</include>
<include>io.netty:netty-codec-http</include>
@@ -197,6 +203,7 @@
<include>io.netty:netty-resolver</include>
<include>io.netty:netty-tcnative-boringssl-static</include>
<include>io.netty:netty-transport</include>

<include>io.opencensus:opencensus-api</include>
<include>io.opencensus:opencensus-contrib-grpc-metrics</include>
<include>io.opencensus:opencensus-contrib-grpc-util</include>
@@ -241,6 +248,7 @@
<include>org.apache.beam:beam-vendor-guava-20_0</include>
<include>org.apache.beam:beam-vendor-guava-26_0-jre</include>
<include>org.apache.beam:beam-vendor-sdks-java-extensions-protobuf</include>
<include>org.apache.hbase:hbase-shaded-client</include>
<include>org.apache.commons:commons-crypto</include>
<include>org.apache.commons:commons-lang3</include>
<include>org.apache.commons:commons-math3</include>
@@ -280,6 +288,7 @@
<include>org.apache.hadoop:hadoop-yarn-client</include>
<include>org.apache.hadoop:hadoop-yarn-common</include>
<include>org.apache.hadoop:hadoop-yarn-server-common</include>
<include>org.apache.htrace:htrace-core</include>
<include>org.apache.ivy:ivy</include>
<include>org.apache.kafka:kafka-clients</include>
<include>org.apache.logging.log4j:log4j-api</include>
@@ -14,7 +14,6 @@
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
@@ -36,7 +35,6 @@

<properties>
<commons-vfs2.version>2.4.1</commons-vfs2.version>
<google-api-client.version>1.30.10</google-api-client.version>
<google-api-sheets.version>v4-rev612-1.25.0</google-api-sheets.version>
<google-api-drive.version>v3-rev197-1.25.0</google-api-drive.version>
<google-api-services-analytics.version>v3-rev169-1.25.0</google-api-services-analytics.version>
@@ -47,6 +45,8 @@
<google-oauth-client.version>1.31.2</google-oauth-client.version>
<google-oauth-client-jetty.version>1.31.2</google-oauth-client-jetty.version>
<jetty-client.version>6.1.26</jetty-client.version>
<grpc.version>1.37.0</grpc.version>
<gax-grpc.version>1.63.0</gax-grpc.version>
</properties>

<dependencies>
@@ -56,6 +56,18 @@
<version>${project.version}</version>
</dependency>

<!-- force lib version to match Beam -->
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-context</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
<version>${gax-grpc.version}</version>
</dependency>

<!-- Google Analytics dependencies -->

<dependency>
@@ -49,10 +49,12 @@
</parent>

<properties>
<apache-beam-version>2.28.0</apache-beam-version>
<apache-beam-version>2.29.0</apache-beam-version>
<spark-version>2.4.7</spark-version>
<flink-version>1.9.3</flink-version>
<flink-version>1.10.1</flink-version>
<json-simple.version>1.1.1</json-simple.version>
<bigtable-hbase-beam.version>1.20.1</bigtable-hbase-beam.version>
<grpc.version>1.37.0</grpc.version>
</properties>

<dependencies>
@@ -115,6 +117,12 @@
<version>2.10.2</version>
</dependency>

<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<version>${google-api-client.version}</version>
</dependency>

<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
@@ -197,7 +205,7 @@
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-flink-1.9</artifactId>
<artifactId>beam-runners-flink-1.10</artifactId>
<version>${apache-beam-version}</version>
<exclusions>
<exclusion>
@@ -399,6 +407,12 @@
<version>${json-simple.version}</version>
</dependency>

<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-beam</artifactId>
<version>${bigtable-hbase-beam.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-memgroupby</artifactId>
@@ -71,6 +71,7 @@ public HopToStringFn(
@Setup
public void setUp() {
try {
initCounter = Metrics.counter(Pipeline.METRIC_NAME_INIT, counterName);
readCounter = Metrics.counter(Pipeline.METRIC_NAME_READ, counterName);
outputCounter = Metrics.counter(Pipeline.METRIC_NAME_OUTPUT, counterName);
errorCounter = Metrics.counter(Pipeline.METRIC_NAME_ERROR, counterName);
@@ -80,7 +81,7 @@ public void setUp() {
BeamHop.init(transformPluginClasses, xpPluginClasses);
rowMeta = JsonRowMeta.fromJson(rowMetaJson);

Metrics.counter(Pipeline.METRIC_NAME_INIT, counterName).inc();
initCounter.inc();
} catch (Exception e) {
errorCounter.inc();
LOG.info("Parse error on setup of Hop data to string lines : " + e.getMessage());

0 comments on commit ef114f0

Please sign in to comment.