Add an example for batch write flow control

GoogleCloudPlatform · May 8, 2024 · 411e662 · 411e662
1 parent bf2cb62
commit 411e662
Show file tree

Hide file tree

Showing 3 changed files with 398 additions and 0 deletions.
diff --git a/bigtable/beam/batch-write-flow-control-example/README.md b/bigtable/beam/batch-write-flow-control-example/README.md
@@ -0,0 +1,43 @@
+# Batch write flow control example
+
+This is an example pipeline to demo how to use the batch write flow control
+feature using CloudBigtableIO and BigtableIO.
+
+## Running instructions
+
+1. Create a Bigtable instance in the console or using gCloud.
+
+1. Create a table with column family `cf`.
+
+1. Set up the environment variables
+
+```
+GOOGLE_CLOUD_PROJECT=<your-project-id>
+INSTANCE_ID=<your-instance-id>
+TABLE_ID=<your-table-id>
+REGION=<your-region>
+NUM_ROWS=<number-of-rows>
+NUM_COLS_PER_ROW=<number-of-columns-per-row>
+NUM_BYTES_PER_COL=<number-of-bytes-per-col>
+NUM_WORKERS=<number-of-workers>
+MAX_NUM_WORKERS=<max-number-of-workers>
+USE_CLOUD_BIGTABLE_IO=<true/false>
+
+```
+
+1. Run the command
+
+```
+mvn compile exec:java -Dexec.mainClass=bigtable.BatchWriteFlowControlExample \
+"-Dexec.args=--runner=dataflow \
+  --project=$PROJECT_ID \
+  --bigtableInstanceId=$INSTANCE_ID \
+  --bigtableTableId=$TABLE_ID \
+  --bigtableRows=$NUM_ROWS \
+  --bigtableColsPerRow=$NUM_COLS_PER_ROW \
+  --bigtableBytesPerCol=$NUM_BYTES_PER_COL\
+  --region=$REGION \
+  --numWorkers=$NUM_WORKERS \
+  --maxNumWorkers=$MAX_NUM_WORKERS \
+  --useCloudBigtableIO=$USE_CLOUD_BIGTABLE_IO"
+```
diff --git a/bigtable/beam/batch-write-flow-control-example/pom.xml b/bigtable/beam/batch-write-flow-control-example/pom.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Copyright 2021 Google LLC
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.example.bigtable</groupId>
+  <artifactId>batch-write-flow-control-example</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <properties>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <apache_beam.version>2.49.0</apache_beam.version>
+  </properties>
+
+  <!--
+  The parent pom defines common style checks and testing strategies for our samples.
+  Removing or replacing it should not affect the execution of the samples in anyway.
+  -->
+  <parent>
+    <groupId>com.google.cloud.samples</groupId>
+    <artifactId>shared-configuration</artifactId>
+    <version>1.2.0</version>
+  </parent>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-bigtable-bom</artifactId>
+        <version>2.22.1-DEBUG-SNAPSHOT</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+      <dependency>
+        <artifactId>libraries-bom</artifactId>
+        <groupId>com.google.cloud</groupId>
+        <scope>import</scope>
+        <type>pom</type>
+        <version>26.22.0</version>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
+      <version>${apache_beam.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-runners-direct-java</artifactId>
+      <version>${apache_beam.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud.bigtable</groupId>
+      <artifactId>bigtable-hbase-beam</artifactId>
+      <version>2.11.0</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-bigtable</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.cloud.bigtable</groupId>
+      <artifactId>bigtable-client-core</artifactId>
+      <version>1.29.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.truth</groupId>
+      <artifactId>truth</artifactId>
+      <version>1.1.5</version>
+      <scope>test</scope>
+    </dependency>
+
+
+  </dependencies>
+
+  <repositories>
+    <repository>
+      <id>artifact-registry</id>
+      <url>artifactregistry://us-maven.pkg.dev/cloud-bigtable-ecosystem/debug-applovin</url>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>
+
+  <build>
+    <extensions>
+      <extension>
+        <groupId>com.google.cloud.artifactregistry</groupId>
+        <artifactId>artifactregistry-maven-wagon</artifactId>
+        <version>2.2.0</version>
+      </extension>
+    </extensions>
+  </build>
+
+</project>