diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/PubsubMessageToTableRow.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/PubsubMessageToTableRow.java index 8322518676..86bc3f69cf 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/PubsubMessageToTableRow.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/PubsubMessageToTableRow.java @@ -49,6 +49,7 @@ public interface TransformOptions @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery output table", + groupName = "Target", helpText = "BigQuery table location to write the output to. The table's schema must match the " + "input JSON objects.") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/ReadFromPubSub.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/ReadFromPubSub.java index 618a2a0ea9..be01b88607 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/ReadFromPubSub.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/ReadFromPubSub.java @@ -32,6 +32,7 @@ public interface ReadFromPubSubOptions extends PipelineOptions { @TemplateParameter.PubsubSubscription( order = 1, description = "Pub/Sub input subscription", + groupName = "Source", helpText = "Pub/Sub subscription to read the input from, in the format of 'projects/your-project-id/subscriptions/your-subscription-name'") String getInputSubscription(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToBigQuery.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToBigQuery.java index 586b2c5ecf..f0e01594a2 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToBigQuery.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToBigQuery.java @@ -62,6 +62,7 @@ public interface SinkOptions @TemplateParameter.BigQueryTable( order = 1, optional = true, + groupName = "Target", description = "BigQuery output table", helpText = "BigQuery table location to write the output to. The table's schema must match the " diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToPubSub.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToPubSub.java index d79780c9d4..f6d64add79 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToPubSub.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/blocks/WriteToPubSub.java @@ -30,6 +30,7 @@ public class WriteToPubSub implements TemplateTransform { public interface WriteToPubSubOptions extends PipelineOptions { @TemplateParameter.PubsubTopic( order = 8, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The name of the topic to which data should published, in the format of 'projects/your-project-id/topics/your-topic-name'", diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/WriteDlqToBigQuery.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/WriteDlqToBigQuery.java index 1bfe7f5507..6d08748833 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/WriteDlqToBigQuery.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/dlq/WriteDlqToBigQuery.java @@ -34,6 +34,7 @@ public interface BigQueryDlqOptions extends PipelineOptions { @TemplateParameter.BigQueryTable( order = 4, optional = true, + groupName = "Target", description = "Table for messages failed to reach the output table (i.e., Deadletter table)", helpText = diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/options/ReadFromJdbcOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/options/ReadFromJdbcOptions.java index 5d3207a045..bc94b15fe1 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/options/ReadFromJdbcOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/auto/options/ReadFromJdbcOptions.java @@ -33,6 +33,7 @@ public interface ReadFromJdbcOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = false, + groupName = "Source", regexes = { "(^jdbc:[a-zA-Z0-9/:@.?_+!*=&-;]+$)|(^([A-Za-z0-9+/]{4}){1,}([A-Za-z0-9+/]{0,3})={0,3})" }, @@ -116,7 +117,6 @@ public interface ReadFromJdbcOptions extends PipelineOptions { @TemplateParameter.Text( order = 9, optional = true, - groupName = "Source Parameters", description = "The name of a column of numeric type that will be used for partitioning.", helpText = "If this parameter is provided (along with `table`), JdbcIO reads the table in parallel by executing multiple instances of the query on the same table (subquery) using ranges. Currently, only Long partition columns are supported.") @@ -127,7 +127,7 @@ public interface ReadFromJdbcOptions extends PipelineOptions { @TemplateParameter.Text( order = 10, optional = true, - groupName = "Source Parameters", + groupName = "Source", description = "Name of the table in the external database.", helpText = "Table to read from using partitions. This parameter also accepts a subquery in parentheses.", @@ -139,7 +139,6 @@ public interface ReadFromJdbcOptions extends PipelineOptions { @TemplateParameter.Integer( order = 11, optional = true, - groupName = "Source Parameters", description = "The number of partitions.", helpText = "The number of partitions. This, along with the lower and upper bound, form partitions strides for generated WHERE clause expressions used to split the partition column evenly. When the input is less than 1, the number is set to 1.") @@ -150,7 +149,6 @@ public interface ReadFromJdbcOptions extends PipelineOptions { @TemplateParameter.Long( order = 12, optional = true, - groupName = "Source Parameters", description = "Lower bound of partition column.", helpText = "Lower bound used in the partition scheme. If not provided, it is automatically inferred by Beam (for the supported types)") @@ -161,7 +159,6 @@ public interface ReadFromJdbcOptions extends PipelineOptions { @TemplateParameter.Long( order = 13, optional = true, - groupName = "Source Parameters", description = "Upper bound of partition column", helpText = "Upper bound used in partition scheme. If not provided, it is automatically inferred by Beam (for the supported types)") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java index e085996fc7..630e4f36a6 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java @@ -36,6 +36,7 @@ public interface WriteOptions extends PipelineOptions { @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery output table", + groupName = "Target", helpText = "The BigQuery output table location to write the output to. For example, `:.`." + "Depending on the `createDisposition` specified, the output table might be created automatically using the user provided Avro schema.") diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/PubsubCommonOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/PubsubCommonOptions.java index ab407a556a..3967957206 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/PubsubCommonOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/PubsubCommonOptions.java @@ -32,6 +32,7 @@ public interface ReadSubscriptionOptions extends PipelineOptions { @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", description = "Pub/Sub input subscription", helpText = "The Pub/Sub input subscription to read from.", example = "projects//subscription/") @@ -62,6 +63,7 @@ public interface WriteTopicOptions extends PipelineOptions { @TemplateParameter.PubsubTopic( order = 3, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The Pub/Sub topic to use for unprocessed records.", example = "projects//topics/") diff --git a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java index 94fe210e07..d940ff401a 100644 --- a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java +++ b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java @@ -27,6 +27,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { @TemplateParameter.Text( order = 1, + groupName = "Target", description = "Elasticsearch URL or CloudID if using Elastic Cloud", helpText = "The Elasticsearch URL in the format https://hostname:[port]. If using Elastic Cloud, specify the CloudID.", diff --git a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/PubSubToElasticsearchOptions.java b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/PubSubToElasticsearchOptions.java index 6945b827cd..bc74f753af 100644 --- a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/PubSubToElasticsearchOptions.java +++ b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/PubSubToElasticsearchOptions.java @@ -36,6 +36,7 @@ public interface PubSubToElasticsearchOptions @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", description = "Pub/Sub input subscription", helpText = "Pub/Sub subscription to consume the input from. Name should be in the format of 'projects/your-project-id/subscriptions/your-subscription-name'", diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/PubsubToJdbcOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/PubsubToJdbcOptions.java index 132fc5a078..a813f01beb 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/PubsubToJdbcOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/PubsubToJdbcOptions.java @@ -26,6 +26,7 @@ public interface PubsubToJdbcOptions extends CommonTemplateOptions { @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", description = "Pub/Sub input subscription", helpText = "The Pub/Sub input subscription to read from, in the format of 'projects//subscriptions/'", @@ -48,6 +49,7 @@ public interface PubsubToJdbcOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 3, + groupName = "Target", optional = false, regexes = { "(^jdbc:[a-zA-Z0-9/:@.?_+!*=&-;]+$)|(^([A-Za-z0-9+/]{4}){1,}([A-Za-z0-9+/]{0,3})={0,3})" diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToBigQueryOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToBigQueryOptions.java index 8737b3b58d..5aa77f880a 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToBigQueryOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToBigQueryOptions.java @@ -31,6 +31,7 @@ public interface SpannerChangeStreamsToBigQueryOptions @TemplateParameter.ProjectId( order = 1, + groupName = "Source", optional = true, description = "Spanner Project ID", helpText = @@ -43,6 +44,7 @@ public interface SpannerChangeStreamsToBigQueryOptions @TemplateParameter.Text( order = 2, + groupName = "Source", description = "Spanner instance ID", helpText = "The Spanner instance to read change streams from.") @Validation.Required @@ -52,6 +54,7 @@ public interface SpannerChangeStreamsToBigQueryOptions @TemplateParameter.Text( order = 3, + groupName = "Source", description = "Spanner database", helpText = "The Spanner database to read change streams from.") @Validation.Required @@ -105,6 +108,7 @@ public interface SpannerChangeStreamsToBigQueryOptions @TemplateParameter.Text( order = 8, + groupName = "Source", description = "Spanner change stream", helpText = "The name of the Spanner change stream to read from.") @Validation.Required @@ -166,6 +170,7 @@ public interface SpannerChangeStreamsToBigQueryOptions @TemplateParameter.Text( order = 13, + groupName = "Target", description = "BigQuery dataset", helpText = "The BigQuery dataset for change streams output.") @Validation.Required @@ -175,6 +180,7 @@ public interface SpannerChangeStreamsToBigQueryOptions @TemplateParameter.ProjectId( order = 14, + groupName = "Target", optional = true, description = "BigQuery project ID", helpText = "The BigQuery project. The default value is the project for the Dataflow job.") @@ -185,6 +191,7 @@ public interface SpannerChangeStreamsToBigQueryOptions @TemplateParameter.Text( order = 15, + groupName = "Target", optional = true, description = "BigQuery table name Template", helpText = "The template for the name of the BigQuery table that contains the changelog.") diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToGcsOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToGcsOptions.java index bc06022508..2667ade3db 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToGcsOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToGcsOptions.java @@ -36,6 +36,7 @@ public interface SpannerChangeStreamsToGcsOptions @TemplateParameter.ProjectId( order = 1, + groupName = "Source", optional = true, description = "Spanner Project ID", helpText = @@ -47,6 +48,7 @@ public interface SpannerChangeStreamsToGcsOptions @TemplateParameter.Text( order = 2, + groupName = "Source", description = "Spanner instance ID", helpText = "The Spanner instance ID to read change streams data from.") @Validation.Required @@ -56,6 +58,7 @@ public interface SpannerChangeStreamsToGcsOptions @TemplateParameter.Text( order = 3, + groupName = "Source", description = "Spanner database", helpText = "The Spanner database to read change streams data from.") @Validation.Required @@ -103,6 +106,7 @@ public interface SpannerChangeStreamsToGcsOptions @TemplateParameter.Text( order = 8, + groupName = "Source", description = "Spanner change stream", helpText = "The name of the Spanner change stream to read from.") @Validation.Required diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java index 41f2dc7591..8de40b8cf5 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java @@ -30,6 +30,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.ProjectId( order = 1, + groupName = "Source", optional = true, description = "Spanner Project ID", helpText = @@ -42,6 +43,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.Text( order = 2, + groupName = "Source", description = "Spanner instance ID", helpText = "The Spanner instance to read change streams from.") @Validation.Required @@ -51,6 +53,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.Text( order = 3, + groupName = "Source", description = "Spanner database", helpText = "The Spanner database to read change streams from.") @Validation.Required @@ -104,6 +107,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.Text( order = 8, + groupName = "Source", description = "Spanner change stream", helpText = "The name of the Spanner change stream to read from.") @Validation.Required @@ -174,6 +178,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.ProjectId( order = 14, + groupName = "Target", optional = true, description = "Pub/Sub Project ID", helpText = @@ -186,6 +191,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.Text( order = 15, + groupName = "Target", description = "The output Pub/Sub topic", helpText = "The Pub/Sub topic for change streams output.") @Validation.Required diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerToBigQueryOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerToBigQueryOptions.java index 3302faaf9a..02b1dc8857 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerToBigQueryOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerToBigQueryOptions.java @@ -27,6 +27,7 @@ public interface SpannerToBigQueryOptions @TemplateParameter.ProjectId( order = 1, + groupName = "Source", optional = true, description = "Spanner Project ID", helpText = @@ -39,6 +40,7 @@ public interface SpannerToBigQueryOptions @TemplateParameter.Text( order = 2, + groupName = "Source", description = "Spanner instance ID", helpText = "The instance ID of the Spanner database to read from.") String getSpannerInstanceId(); @@ -47,6 +49,7 @@ public interface SpannerToBigQueryOptions @TemplateParameter.Text( order = 3, + groupName = "Source", description = "Spanner database ID", helpText = "The database ID of the Spanner database to export.") String getSpannerDatabaseId(); @@ -55,6 +58,7 @@ public interface SpannerToBigQueryOptions @TemplateParameter.Text( order = 4, + groupName = "Source", description = "Spanner table name", helpText = "The table name of the Spanner database to export.") String getSpannerTableId(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToBigQuery.java index 4acbaf609f..a88f14f363 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToBigQuery.java @@ -181,6 +181,7 @@ public interface Options DataflowPipelineWorkerPoolOptions { @TemplateParameter.BigQueryTable( order = 1, + groupName = "Target", description = "BigQuery output table", helpText = "The BigQuery table to write to, formatted as `\"PROJECT_ID:DATASET_NAME.TABLE_NAME\"`.") @@ -190,6 +191,7 @@ public interface Options @TemplateParameter.PubsubTopic( order = 2, + groupName = "Source", optional = true, description = "Input Pub/Sub topic", helpText = @@ -200,6 +202,7 @@ public interface Options @TemplateParameter.PubsubSubscription( order = 3, + groupName = "Source", optional = true, description = "Pub/Sub input subscription", helpText = diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToAvro.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToAvro.java index 09fb8aed56..767fb94dc4 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToAvro.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToAvro.java @@ -90,6 +90,7 @@ public interface Options extends PipelineOptions, StreamingOptions, WindowedFilenamePolicyOptions { @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", optional = true, description = "Pub/Sub input subscription", helpText = @@ -102,6 +103,7 @@ public interface Options @TemplateParameter.PubsubTopic( order = 2, + groupName = "Source", optional = true, description = "Pub/Sub input topic", helpText = @@ -113,6 +115,7 @@ public interface Options @TemplateParameter.GcsWriteFolder( order = 3, + groupName = "Target", optional = false, description = "Output file directory in Cloud Storage", helpText = @@ -125,6 +128,7 @@ public interface Options @TemplateParameter.Text( order = 4, + groupName = "Target", optional = true, description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", @@ -136,6 +140,7 @@ public interface Options @TemplateParameter.Text( order = 5, + groupName = "Target", optional = true, description = "Output filename suffix of the files to write", helpText = diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index 21f4418d37..c6f0abc132 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -212,6 +212,7 @@ public interface Options BigQueryStorageApiBatchOptions { @TemplateParameter.GcsReadFile( order = 1, + groupName = "Source", optional = false, description = "The GCS location of the text you'd like to process", helpText = "The gs:// path to the text in Cloud Storage you'd like to process.", @@ -234,6 +235,7 @@ public interface Options @TemplateParameter.BigQueryTable( order = 3, optional = false, + groupName = "Target", description = "Output table to write to", helpText = "The location of the BigQuery table to use to store the processed data. If you reuse an existing table, it is overwritten.", diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/pubsubtotext/PubsubToText.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/pubsubtotext/PubsubToText.java index 94ac696d7c..0b66afd011 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/pubsubtotext/PubsubToText.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/pubsubtotext/PubsubToText.java @@ -77,6 +77,7 @@ public interface Options @TemplateParameter.PubsubTopic( order = 1, + groupName = "Source", optional = true, description = "Pub/Sub input topic", helpText = @@ -90,6 +91,7 @@ public interface Options @TemplateParameter.PubsubSubscription( order = 2, + groupName = "Source", optional = true, description = "Pub/Sub input subscription", helpText = @@ -103,6 +105,7 @@ public interface Options @TemplateParameter.GcsWriteFolder( order = 3, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix to write write output files to. " @@ -125,6 +128,7 @@ public interface Options @TemplateParameter.Text( order = 5, + groupName = "Target", optional = true, description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", @@ -137,6 +141,7 @@ public interface Options @TemplateParameter.Text( order = 6, + groupName = "Target", optional = true, description = "Output filename suffix of the files to write", helpText = diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsAvro.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsAvro.java index a2784ca4d7..f2c7247c82 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsAvro.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsAvro.java @@ -96,6 +96,7 @@ public PDone expand(PCollection dataChangeRecords) { public interface WriteToGcsAvroOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash. DateTime" @@ -108,6 +109,7 @@ public interface WriteToGcsAvroOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, optional = true, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-") diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsText.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsText.java index 64db218fdd..efc76b8a02 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsText.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/transforms/WriteDataChangeRecordsToGcsText.java @@ -100,6 +100,7 @@ public PDone expand(PCollection dataChangeRecords) { public interface WriteToGcsTextOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 1, + groupName = "Target", description = "Output file directory in Cloud Storage", helpText = "The path and filename prefix for writing output files. Must end with a slash. DateTime formatting is used to parse directory path for date & time formatters.", @@ -110,6 +111,7 @@ public interface WriteToGcsTextOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, + groupName = "Target", description = "Output filename prefix of the files to write", helpText = "The prefix to place on each windowed file.", example = "output-") diff --git a/v2/kafka-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KafkaToPubsubOptions.java b/v2/kafka-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KafkaToPubsubOptions.java index db7a28e2d4..71602eca71 100644 --- a/v2/kafka-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KafkaToPubsubOptions.java +++ b/v2/kafka-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KafkaToPubsubOptions.java @@ -29,6 +29,7 @@ public interface KafkaToPubsubOptions extends PipelineOptions, KafkaCommonOptions, JavascriptTextTransformerOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", optional = true, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", @@ -41,6 +42,7 @@ public interface KafkaToPubsubOptions @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "Kafka topic(s) to read the input from", @@ -53,6 +55,7 @@ public interface KafkaToPubsubOptions @TemplateParameter.PubsubTopic( order = 3, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The name of the topic to which data should published, in the format of 'projects/your-project-id/topics/your-topic-name'", diff --git a/v2/kinesis-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KinesisToPubsubOptions.java b/v2/kinesis-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KinesisToPubsubOptions.java index 2b037b1f38..c9d732b75c 100644 --- a/v2/kinesis-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KinesisToPubsubOptions.java +++ b/v2/kinesis-to-pubsub/src/main/java/com/google/cloud/teleport/v2/options/KinesisToPubsubOptions.java @@ -72,6 +72,7 @@ public interface KinesisToPubsubOptions @TemplateParameter.Text( order = 5, + groupName = "Source", optional = false, description = "Name of the Kinesis Data stream to read from", helpText = @@ -84,6 +85,7 @@ public interface KinesisToPubsubOptions @TemplateParameter.PubsubTopic( order = 6, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The name of the topic to which data should published, " diff --git a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java index 32786d7c29..d228b30ec1 100644 --- a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java +++ b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java @@ -31,6 +31,7 @@ public class MongoDbToBigQueryOptions { public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", description = "MongoDB Connection URI", helpText = "The MongoDB connection URI in the format `mongodb+srv://:@.`") String getMongoDbUri(); @@ -39,6 +40,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 2, + groupName = "Source", description = "MongoDB database", helpText = "Database in MongoDB to read the collection from.", example = "my-db") @@ -48,6 +50,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions @TemplateParameter.Text( order = 3, + groupName = "Source", description = "MongoDB collection", helpText = "Name of the collection inside MongoDB database.", example = "my-collection") @@ -84,6 +87,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions public interface PubSubOptions extends PipelineOptions, DataflowPipelineOptions { @TemplateParameter.PubsubTopic( order = 1, + groupName = "Source", description = "Pub/Sub input topic", helpText = "The Pub/Sub input topic to read from, in the format of projects//topics/.") @@ -97,6 +101,7 @@ public interface BigQueryWriteOptions extends PipelineOptions, DataflowPipelineO @TemplateParameter.BigQueryTable( order = 1, + groupName = "Target", description = "BigQuery output table", helpText = "The BigQuery table to write to. For example, `bigquery-project:dataset.output_table`.") diff --git a/v2/mqtt-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/MqttToPubsub.java b/v2/mqtt-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/MqttToPubsub.java index 9ab1f45b0c..070a756455 100644 --- a/v2/mqtt-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/MqttToPubsub.java +++ b/v2/mqtt-to-pubsub/src/main/java/com/google/cloud/teleport/v2/templates/MqttToPubsub.java @@ -121,6 +121,7 @@ public void processElement(@Element byte[] word, OutputReceiver out) { public interface MqttToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", optional = true, regexes = {"[,\\/:a-zA-Z0-9._-]+"}, description = "MQTT Broker IP", @@ -133,6 +134,7 @@ public interface MqttToPubsubOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, regexes = {"[\\/a-zA-Z0-9._-]+"}, description = "MQTT topic(s) to read the input from", @@ -145,6 +147,7 @@ public interface MqttToPubsubOptions extends PipelineOptions { @TemplateParameter.PubsubTopic( order = 3, + groupName = "Target", description = "Output Pub/Sub topic", helpText = "The name of the output Pub/Sub topic that data is written to.", example = "projects/your-project-id/topics/your-topic-name") diff --git a/v2/pubsub-cdc-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/PubSubCdcToBigQuery.java b/v2/pubsub-cdc-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/PubSubCdcToBigQuery.java index c6cfe8bec8..88844e90f3 100644 --- a/v2/pubsub-cdc-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/PubSubCdcToBigQuery.java +++ b/v2/pubsub-cdc-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/PubSubCdcToBigQuery.java @@ -132,6 +132,7 @@ public interface Options @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", description = "Pub/Sub input subscription", helpText = "Pub/Sub subscription to read the input from, in the format of" @@ -166,6 +167,7 @@ public interface Options @TemplateParameter.Text( order = 4, + groupName = "Target", description = "BigQuery Dataset Name or Template: dataset_name or {column_name}", helpText = "The name for the dataset to contain the replica table.") @Default.String("{_metadata_dataset}") @@ -175,6 +177,7 @@ public interface Options @TemplateParameter.Text( order = 5, + groupName = "Target", description = "BigQuery Table Name or Template: table_name or {column_name}", helpText = "The location of the BigQuery table to write the output to. If a table does not " @@ -186,6 +189,7 @@ public interface Options @TemplateParameter.BigQueryTable( order = 6, + groupName = "Target", optional = true, description = "BigQuery output table (Deprecated)", helpText = diff --git a/v2/pubsub-to-jms/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToJms.java b/v2/pubsub-to-jms/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToJms.java index b78c8d57ec..b31ebb7f0c 100644 --- a/v2/pubsub-to-jms/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToJms.java +++ b/v2/pubsub-to-jms/src/main/java/com/google/cloud/teleport/v2/templates/PubsubToJms.java @@ -128,6 +128,7 @@ public static PipelineResult run(PubsubToJmsOptions options) { public interface PubsubToJmsOptions extends PipelineOptions { @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", description = "Pub/Sub input subscription", helpText = "Pub/Sub subscription to read the input from, in the format of" @@ -140,6 +141,7 @@ public interface PubsubToJmsOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, + groupName = "Target", optional = true, regexes = {"[,\\/:a-zA-Z0-9._-]+"}, description = "JMS Host IP", @@ -152,6 +154,7 @@ public interface PubsubToJmsOptions extends PipelineOptions { @TemplateParameter.Text( order = 3, + groupName = "Target", optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "JMS Queue/Topic Name to write the input to", diff --git a/v2/pubsub-to-kafka/src/main/java/com/google/cloud/teleport/v2/options/PubsubToKafkaOptions.java b/v2/pubsub-to-kafka/src/main/java/com/google/cloud/teleport/v2/options/PubsubToKafkaOptions.java index 3e2e2afb44..9de9885a32 100644 --- a/v2/pubsub-to-kafka/src/main/java/com/google/cloud/teleport/v2/options/PubsubToKafkaOptions.java +++ b/v2/pubsub-to-kafka/src/main/java/com/google/cloud/teleport/v2/options/PubsubToKafkaOptions.java @@ -29,6 +29,7 @@ public interface PubsubToKafkaOptions extends PipelineOptions, KafkaCommonOptions, JavascriptTextTransformerOptions { @TemplateParameter.PubsubTopic( order = 1, + groupName = "Source", description = "Input Pub/Sub topic", helpText = "The name of the topic from which data should published, in the format of " @@ -41,6 +42,7 @@ public interface PubsubToKafkaOptions @TemplateParameter.Text( order = 2, + groupName = "Target", optional = true, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Output Kafka Bootstrap Server", @@ -53,6 +55,7 @@ public interface PubsubToKafkaOptions @TemplateParameter.Text( order = 3, + groupName = "Target", optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "Kafka topic to write the input from pubsub", diff --git a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java index ebe2545385..da89e0b4b2 100644 --- a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java +++ b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java @@ -171,6 +171,7 @@ public interface Options extends PythonExternalTextTransformer.PythonExternalTextTransformerOptions, PipelineOptions { @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", description = "Pub/Sub input subscription", helpText = "Name of the Pub/Sub subscription.", example = "projects/your-project-id/subscriptions/your-subscription-name") @@ -181,6 +182,7 @@ public interface Options @TemplateParameter.Text( order = 2, + groupName = "Target", description = "MongoDB Connection URI", helpText = "Comma separated list of MongoDB servers.", example = "host1:port,host2:port,host3:port") @@ -191,6 +193,7 @@ public interface Options @TemplateParameter.Text( order = 3, + groupName = "Target", description = "MongoDB Database", helpText = "Database in MongoDB to store the collection.", example = "my-db") @@ -201,6 +204,7 @@ public interface Options @TemplateParameter.Text( order = 4, + groupName = "Target", description = "MongoDB collection", helpText = "Name of the collection in the MongoDB database.", example = "my-collection") diff --git a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java index b2b14f3c2c..25db3a35c1 100644 --- a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java +++ b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java @@ -133,6 +133,7 @@ public interface PubSubToRedisOptions extends JavascriptTextTransformer.JavascriptTextTransformerOptions, PipelineOptions { @TemplateParameter.PubsubSubscription( order = 1, + groupName = "Source", description = "Pub/Sub input subscription", helpText = "The Pub/Sub subscription to read the input from, in the format" @@ -144,6 +145,7 @@ public interface PubSubToRedisOptions @TemplateParameter.Text( order = 2, + groupName = "Target", description = "Redis DB Host", helpText = "The Redis database host.", example = "your.cloud.db.redislabs.com") @@ -155,6 +157,7 @@ public interface PubSubToRedisOptions @TemplateParameter.Integer( order = 3, + groupName = "Target", description = "Redis DB Port", helpText = "The Redis database port.", example = "12345") @@ -166,6 +169,7 @@ public interface PubSubToRedisOptions @TemplateParameter.Password( order = 4, + groupName = "Target", description = "Redis DB Password", helpText = "The Redis database password. Defaults to empty.") @Default.String("") diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/SourceDbToSpannerOptions.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/SourceDbToSpannerOptions.java index 624fb15786..d8e2291bdc 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/SourceDbToSpannerOptions.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/options/SourceDbToSpannerOptions.java @@ -25,7 +25,6 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { order = 1, optional = true, regexes = {"^.+$"}, - groupName = "Source", description = "Comma-separated Cloud Storage path(s) of the JDBC driver(s)", helpText = "The comma-separated list of driver JAR files.", example = "gs://your-bucket/driver_jar1.jar,gs://your-bucket/driver_jar2.jar") @@ -38,7 +37,6 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { order = 2, optional = true, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC driver class name", helpText = "The JDBC driver class name.", example = "com.mysql.jdbc.Driver") @@ -63,7 +61,6 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { order = 4, optional = true, regexes = {"^.+$"}, - groupName = "Source", description = "JDBC connection username.", helpText = "The username to be used for the JDBC connection.") @Default.String("") @@ -74,7 +71,6 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { @TemplateParameter.Password( order = 5, optional = true, - groupName = "Source", description = "JDBC connection password.", helpText = "The password to be used for the JDBC connection.") @Default.String("") @@ -85,7 +81,6 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 6, optional = true, - groupName = "Source Parameters", description = "Comma-separated names of the tables in the source database.", helpText = "Tables to read from using partitions.") @Default.String("") @@ -97,7 +92,6 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { @TemplateParameter.Integer( order = 7, optional = true, - groupName = "Source", description = "The number of partitions.", helpText = "The number of partitions. This, along with the lower and upper bound, form partitions" @@ -110,6 +104,7 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 8, + groupName = "Target", description = "Cloud Spanner Instance Id.", helpText = "The destination Cloud Spanner instance.") String getInstanceId(); @@ -118,6 +113,7 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { @TemplateParameter.Text( order = 9, + groupName = "Target", regexes = {"^[a-z]([a-z0-9_-]{0,28})[a-z0-9]$"}, description = "Cloud Spanner Database Id.", helpText = "The destination Cloud Spanner database.") @@ -127,6 +123,7 @@ public interface SourceDbToSpannerOptions extends CommonTemplateOptions { @TemplateParameter.ProjectId( order = 10, + groupName = "Target", description = "Cloud Spanner Project Id.", helpText = "This is the name of the Cloud Spanner project.") String getProjectId(); diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java index 084e2f8de5..ed650d4727 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java @@ -95,6 +95,7 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 1, + groupName = "Source", optional = false, description = "Name of the change stream to read from", helpText = @@ -105,6 +106,7 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 2, + groupName = "Source", optional = false, description = "Cloud Spanner Instance Id.", helpText = @@ -115,6 +117,7 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.Text( order = 3, + groupName = "Source", optional = false, description = "Cloud Spanner Database Id.", helpText = @@ -125,6 +128,7 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.ProjectId( order = 4, + groupName = "Source", optional = false, description = "Cloud Spanner Project Id.", helpText = "This is the name of the Cloud Spanner project.") @@ -202,6 +206,7 @@ public interface Options extends PipelineOptions, StreamingOptions { @TemplateParameter.GcsWriteFolder( order = 11, + groupName = "Target", optional = false, description = "Output file directory in Cloud Storage", helpText =