Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add groupName attribute to v1 template where it's needed (part 3) #1542

Merged
merged 2 commits into from
Jun 6, 2024
Merged

Add groupName attribute to v1 template where it's needed (part 3) #1542

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f46114
Add groupName attribute to v1 template where it's needed (part 3)
akashorabek May 10, 2024
02a78dd
Resolve comments
Amar3tto May 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/bigtable/AvroToBigtable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ public final class AvroToBigtable {
public interface Options extends PipelineOptions {
@TemplateParameter.ProjectId(
order = 1,
groupName = "Target",
description = "Project ID",
helpText =
"The ID of the Google Cloud project that contains the Bigtable instance that you want to write data to.")
Expand All @@ -88,6 +89,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 2,
groupName = "Target",
regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"},
description = "Instance ID",
helpText = "The ID of the Bigtable instance that contains the table.")
Expand All @@ -98,6 +100,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 4,
groupName = "Target",
regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"},
description = "Table ID",
helpText = "The ID of the Bigtable table to import.")
Expand All @@ -108,6 +111,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.GcsReadFile(
order = 5,
groupName = "Source",
description = "Input Cloud Storage File(s)",
helpText = "The Cloud Storage path pattern where data is located.",
example = "gs://<BUCKET>/<FOLDER>/<PREFIX>*")
Expand Down
5 changes: 5 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToAvro.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public class BigtableToAvro {
public interface Options extends PipelineOptions {
@TemplateParameter.ProjectId(
order = 1,
groupName = "Source",
description = "Project ID",
helpText =
"The ID of the Google Cloud project that contains the Bigtable instance that you want to read data from.")
Expand All @@ -86,6 +87,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 2,
groupName = "Source",
regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"},
description = "Instance ID",
helpText = "The ID of the Bigtable instance that contains the table.")
Expand All @@ -96,6 +98,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 3,
groupName = "Source",
regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"},
description = "Table ID",
helpText = "The ID of the Bigtable table to export.")
Expand All @@ -106,6 +109,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.GcsWriteFolder(
order = 4,
groupName = "Target",
description = "Output file directory in Cloud Storage",
helpText = "The Cloud Storage path where data is written.",
example = "gs://mybucket/somefolder")
Expand All @@ -116,6 +120,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 5,
groupName = "Target",
description = "Avro file prefix",
helpText = "The prefix of the Avro filename. For example, `output-`.")
@Default.String("part")
Expand Down
7 changes: 7 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public class BigtableToJson {
public interface Options extends PipelineOptions {
@TemplateParameter.ProjectId(
order = 1,
groupName = "Source",
description = "Project ID",
helpText =
"The ID for the Google Cloud project that contains the Bigtable instance that you want to read data from.")
Expand All @@ -84,6 +85,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 2,
groupName = "Source",
regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"},
description = "Instance ID",
helpText = "The ID of the Bigtable instance that contains the table.")
Expand All @@ -94,6 +96,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 3,
groupName = "Source",
regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"},
description = "Table ID",
helpText = "The ID of the Bigtable table to read from.")
Expand All @@ -104,6 +107,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.GcsWriteFolder(
order = 4,
groupName = "Target",
optional = true,
description = "Cloud Storage directory for storing JSON files",
helpText = "The Cloud Storage path where the output JSON files are stored.",
Expand All @@ -115,6 +119,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 5,
groupName = "Target",
description = "JSON file prefix",
helpText =
"The prefix of the JSON file name. For example, \"table1-\". If no value is provided, defaults to `part`.")
Expand All @@ -126,6 +131,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Enum(
order = 6,
groupName = "Target",
optional = true,
enumOptions = {@TemplateEnumOption("FLATTEN"), @TemplateEnumOption("NONE")},
description = "User option",
Expand All @@ -139,6 +145,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 7,
groupName = "Target",
optional = true,
parentName = "userOption",
parentTriggerValues = {"FLATTEN"},
Expand Down
6 changes: 6 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToParquet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.ProjectId(
order = 1,
groupName = "Source",
description = "Project ID",
helpText =
"The ID of the Google Cloud project that contains the Cloud Bigtable instance that you want to read data from.")
Expand All @@ -85,6 +86,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 2,
groupName = "Source",
regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"},
description = "Instance ID",
helpText = "The ID of the Cloud Bigtable instance that contains the table.")
Expand All @@ -95,6 +97,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 3,
groupName = "Source",
regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"},
description = "Table ID",
helpText = "The ID of the Cloud Bigtable table to export.")
Expand All @@ -105,6 +108,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.GcsWriteFolder(
order = 4,
groupName = "Target",
description = "Output file directory in Cloud Storage",
helpText =
"The path and filename prefix for writing output files. Must end with a slash. DateTime formatting is used to parse the directory path for date and time formatters. For example: gs://your-bucket/your-path.")
Expand All @@ -115,6 +119,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 5,
groupName = "Target",
description = "Parquet file prefix",
helpText =
"The prefix of the Parquet file name. For example, \"table1-\". Defaults to: part.")
Expand All @@ -126,6 +131,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Integer(
order = 6,
groupName = "Target",
optional = true,
description = "Maximum output shards",
helpText =
Expand Down
5 changes: 5 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToVectorEmbeddings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public class BigtableToVectorEmbeddings {
public interface Options extends PipelineOptions {
@TemplateParameter.ProjectId(
order = 1,
groupName = "Source",
description = "Project ID",
helpText =
"The ID for the Google Cloud project that contains the Bigtable instance that you want to read data from.")
Expand All @@ -90,6 +91,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 2,
groupName = "Source",
regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"},
description = "Instance ID",
helpText = "The ID of the Bigtable instance that contains the table.")
Expand All @@ -100,6 +102,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 3,
groupName = "Source",
regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"},
description = "Table ID",
helpText = "The ID of the Bigtable table to read from.")
Expand All @@ -110,6 +113,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.GcsWriteFolder(
order = 4,
groupName = "Target",
optional = true,
description = "Cloud Storage directory for storing JSON files",
helpText = "The Cloud Storage path where the output JSON files are stored.",
Expand All @@ -121,6 +125,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 5,
groupName = "Target",
description = "JSON file prefix",
helpText =
"The prefix of the JSON filename. For example: \"table1-\". If no value is provided, defaults to \"part\".")
Expand Down
10 changes: 10 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/bigtable/CassandraToBigtable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 1,
groupName = "Source",
regexes = {"^[a-zA-Z0-9\\.\\-,]*$"},
description = "Cassandra Hosts",
helpText = "The hosts of the Apache Cassandra nodes in a comma-separated list.")
Expand All @@ -80,6 +81,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Integer(
order = 2,
groupName = "Source",
optional = true,
description = "Cassandra Port",
helpText =
Expand All @@ -92,6 +94,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 3,
groupName = "Source",
regexes = {"^[a-zA-Z0-9][a-zA-Z0-9_]{0,47}$"},
description = "Cassandra Keyspace",
helpText = "The Apache Cassandra keyspace where the table is located.")
Expand All @@ -102,6 +105,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 4,
groupName = "Source",
regexes = {"^[a-zA-Z][a-zA-Z0-9_]*$"},
description = "Cassandra Table",
helpText = "The Apache Cassandra table to copy.")
Expand All @@ -112,6 +116,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.ProjectId(
order = 5,
groupName = "Target",
description = "Bigtable Project ID",
helpText = "The Google Cloud project ID associated with the Bigtable instance.")
ValueProvider<String> getBigtableProjectId();
Expand All @@ -121,6 +126,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 6,
groupName = "Target",
regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"},
description = "Target Bigtable Instance",
helpText = "The ID of the Bigtable instance that the Apache Cassandra table is copied to.")
Expand All @@ -131,6 +137,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 7,
groupName = "Target",
regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"},
description = "Target Bigtable Table",
helpText = "The name of the Bigtable table that the Apache Cassandra table is copied to.")
Expand All @@ -141,6 +148,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 8,
groupName = "Target",
optional = true,
regexes = {"[-_.a-zA-Z0-9]+"},
description = "The Default Bigtable Column Family",
Expand All @@ -154,6 +162,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 9,
groupName = "Target",
optional = true,
description = "The Row Key Separator",
helpText = "The separator used to build row-keys. The default value is '#'.")
Expand All @@ -165,6 +174,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Boolean(
order = 10,
groupName = "Target",
optional = true,
description = "If true, large rows will be split into multiple MutateRows requests",
helpText =
Expand Down
5 changes: 5 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/bigtable/ParquetToBigtable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ public class ParquetToBigtable {
public interface Options extends PipelineOptions {
@TemplateParameter.ProjectId(
order = 1,
groupName = "Target",
description = "Project ID",
helpText = "The Google Cloud project ID associated with the Bigtable instance.")
ValueProvider<String> getBigtableProjectId();
Expand All @@ -91,6 +92,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 2,
groupName = "Target",
regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"},
description = "Instance ID",
helpText = "The ID of the Cloud Bigtable instance that contains the table")
Expand All @@ -101,6 +103,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Text(
order = 3,
groupName = "Target",
regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"},
description = "Table ID",
helpText = "The ID of the Bigtable table to import.")
Expand All @@ -111,6 +114,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.GcsReadFile(
order = 4,
groupName = "Source",
description = "Input Cloud Storage File(s)",
helpText = "The Cloud Storage path with the files that contain the data.",
example = "gs://your-bucket/your-files/*.parquet")
Expand All @@ -121,6 +125,7 @@ public interface Options extends PipelineOptions {

@TemplateParameter.Boolean(
order = 5,
groupName = "Target",
optional = true,
description = "If true, large rows will be split into multiple MutateRows requests",
helpText =
Expand Down
Loading
Loading