From 21fd11470b28b8a0ad009830c3341465876366f0 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 26 Apr 2024 11:10:19 +0530 Subject: [PATCH 01/16] Modified parameters for testing --- .../google/cloud/teleport/v2/templates/BigQueryToBigtable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/bigquery-to-bigtable/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToBigtable.java b/v2/bigquery-to-bigtable/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToBigtable.java index 18f09d0390..87575af0d2 100644 --- a/v2/bigquery-to-bigtable/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToBigtable.java +++ b/v2/bigquery-to-bigtable/src/main/java/com/google/cloud/teleport/v2/templates/BigQueryToBigtable.java @@ -81,7 +81,7 @@ public interface BigQueryToBigtableOptions order = 1, regexes = {"[A-Za-z_][A-Za-z_0-9]*"}, description = "Unique identifier column", - helpText = "Name of the BigQuery column storing the unique identifier of the row") + helpText = "The name of the BigQuery column storing the unique identifier of the row.") @Required String getReadIdColumn(); From 0c3e37efca27a7db269069dbff8997c3b7173a5d Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Wed, 8 May 2024 01:21:13 +0530 Subject: [PATCH 02/16] Modified parameters for Dataflow templates --- .../teleport/bigtable/BigtableToJson.java | 24 +++++----- .../WindowedFilenamePolicyOptions.java | 8 ++-- .../teleport/templates/CSVToBigQuery.java | 37 ++++---------- .../teleport/templates/PubsubToAvro.java | 13 ++--- .../teleport/templates/SpannerToText.java | 5 +- .../templates/common/CsvConverters.java | 10 ++-- .../templates/common/SpannerConverters.java | 21 ++++---- .../templates/common/TextConverters.java | 2 +- .../options/BigtableCommonOptions.java | 26 +++++----- .../v2/options/BigQueryCommonOptions.java | 3 +- .../BigQueryStorageApiStreamingOptions.java | 34 ++++++------- .../v2/options/CommonTemplateOptions.java | 3 +- .../v2/transforms/BigQueryConverters.java | 18 ++++--- .../teleport/v2/transforms/CsvConverters.java | 9 ++-- .../transforms/JavascriptTextTransformer.java | 7 ++- .../options/ElasticsearchWriteOptions.java | 38 +++++++-------- .../options/GCSToElasticsearchOptions.java | 2 +- .../SpannerChangeStreamsToPubSubOptions.java | 48 +++++++++---------- .../v2/options/JdbcToPubsubOptions.java | 21 ++++---- .../v2/templates/KafkaToBigQuery.java | 4 +- .../options/MongoDbToBigQueryOptions.java | 15 +++--- .../templates/MongoDbToBigQueryCdc.java | 7 +-- .../teleport/v2/templates/PubSubToRedis.java | 18 +++---- 23 files changed, 167 insertions(+), 206 deletions(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java b/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java index 9d77f78883..966d8e0fbe 100644 --- a/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java +++ b/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java @@ -76,8 +76,7 @@ public interface Options extends PipelineOptions { order = 1, description = "Project ID", helpText = - "The ID of the Google Cloud project of the Cloud Bigtable instance that you want to" - + " read data from") + "The ID for the Google Cloud project that contains the Bigtable instance that you want to read data from.") ValueProvider getBigtableProjectId(); @SuppressWarnings("unused") @@ -87,7 +86,7 @@ public interface Options extends PipelineOptions { order = 2, regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Instance ID", - helpText = "The ID of the Cloud Bigtable instance that contains the table") + helpText = "The ID of the Bigtable instance that contains the table") ValueProvider getBigtableInstanceId(); @SuppressWarnings("unused") @@ -97,7 +96,7 @@ public interface Options extends PipelineOptions { order = 3, regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"}, description = "Table ID", - helpText = "The ID of the Cloud Bigtable table to read") + helpText = "The ID of the Bigtable table to read from.") ValueProvider getBigtableTableId(); @SuppressWarnings("unused") @@ -107,7 +106,7 @@ public interface Options extends PipelineOptions { order = 4, optional = true, description = "Cloud Storage directory for storing JSON files", - helpText = "The Cloud Storage path where the output JSON files can be stored.", + helpText = "The Cloud Storage path where the output JSON files are stored.", example = "gs://your-bucket/your-path/") ValueProvider getOutputDirectory(); @@ -117,7 +116,7 @@ public interface Options extends PipelineOptions { @TemplateParameter.Text( order = 5, description = "JSON file prefix", - helpText = "The prefix of the JSON file name. For example, \"table1-\"") + helpText = "The prefix of the JSON file name. For example, \"table1-\". If no value is provided, defaults to `part`.") @Default.String("part") ValueProvider getFilenamePrefix(); @@ -130,7 +129,7 @@ public interface Options extends PipelineOptions { enumOptions = {@TemplateEnumOption("FLATTEN"), @TemplateEnumOption("NONE")}, description = "User option", helpText = - "User option: `FLATTEN` or `NONE`. `FLATTEN` flattens the row to the single level. `NONE` stores the whole row as a JSON string.") + "Possible values are `FLATTEN` or `NONE`. `FLATTEN` flattens the row to the single level. `NONE` stores the whole row as a JSON string. Defaults to `NONE`.") @Default.String("NONE") String getUserOption(); @@ -142,12 +141,11 @@ public interface Options extends PipelineOptions { optional = true, description = "Columns aliases", helpText = - "Comma separated list of columns which are required for Vertex AI Vector Search Index." - + " The `id` & `embedding` are required columns for Vertex Vector Search." - + " You can use the notation `fromfamily:fromcolumn;to`. For example, if the columns are" - + " `rowkey` and `cf:my_embedding`, in which `rowkey` and the embedding column is named differently," - + " `cf:my_embedding;embedding` and `rowkey;id` should be specified." - + " Only used when FLATTEN user option is specified.") + "A comma-separated list of columns that are required for the Vertex AI Vector Search index. The" + + " columns `id` and `embedding` are required for Vertex AI Vector Search. You can use the notation" + + " `fromfamily:fromcolumn;to`. For example, if the columns are `rowkey` and `cf:my_embedding`, where" + + " `rowkey` has a different name than the embedding column, specify `cf:my_embedding;embedding` and," + + " `rowkey;id`. Only use this option when the value for `userOption` is `FLATTEN`.") ValueProvider getColumnsAliases(); @SuppressWarnings("unused") diff --git a/v1/src/main/java/com/google/cloud/teleport/options/WindowedFilenamePolicyOptions.java b/v1/src/main/java/com/google/cloud/teleport/options/WindowedFilenamePolicyOptions.java index d3446fd702..2ad51b73ac 100644 --- a/v1/src/main/java/com/google/cloud/teleport/options/WindowedFilenamePolicyOptions.java +++ b/v1/src/main/java/com/google/cloud/teleport/options/WindowedFilenamePolicyOptions.java @@ -33,11 +33,9 @@ public interface WindowedFilenamePolicyOptions extends PipelineOptions { optional = true, description = "Shard template", helpText = - "Defines the unique/dynamic portion of each windowed file. Recommended: use the default" - + " (W-P-SS-of-NN). At runtime, 'W' is replaced with the window date range and 'P' is" - + " replaced with the pane info. Repeating sequences of the letters 'S' or 'N' are" - + " replaced with the shard number and number of shards respectively. The pipeline" - + " assumes a single file output and will produce the text of '00-of-01' by default.", + "The shard template of the output file. It is specified as repeating sequences of the letters `S` or" + + " `N`. For example, `SSS-NNN`. These are replaced with either the shard number or the total number of shards," + + " respectively. When this parameter is not specified, the default template format is `W-P-SS-of-NN`.", regexes = "^W-P-(S){1,}-of-(N){1,}$") @Default.String("W-P-SS-of-NN") ValueProvider getOutputShardTemplate(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java b/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java index 8175f474df..84d9dd32ea 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java @@ -102,7 +102,7 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { @TemplateParameter.Text( order = 1, description = "Cloud Storage Input File(s)", - helpText = "Path of the file pattern glob to read from.", + helpText = "The Cloud Storage path to the CSV file that contains the text to process.", regexes = {"^gs:\\/\\/[^\\n\\r]+$"}, example = "gs://your-bucket/path/*.csv") ValueProvider getInputFilePattern(); @@ -113,30 +113,7 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { order = 2, description = "Cloud Storage location of your BigQuery schema file, described as a JSON", helpText = - "JSON file with BigQuery Schema description. JSON Example: {\n" - + "\t\"BigQuery Schema\": [\n" - + "\t\t{\n" - + "\t\t\t\"name\": \"location\",\n" - + "\t\t\t\"type\": \"STRING\"\n" - + "\t\t},\n" - + "\t\t{\n" - + "\t\t\t\"name\": \"name\",\n" - + "\t\t\t\"type\": \"STRING\"\n" - + "\t\t},\n" - + "\t\t{\n" - + "\t\t\t\"name\": \"age\",\n" - + "\t\t\t\"type\": \"STRING\"\n" - + "\t\t},\n" - + "\t\t{\n" - + "\t\t\t\"name\": \"color\",\n" - + "\t\t\t\"type\": \"STRING\"\n" - + "\t\t},\n" - + "\t\t{\n" - + "\t\t\t\"name\": \"coffee\",\n" - + "\t\t\t\"type\": \"STRING\"\n" - + "\t\t}\n" - + "\t]\n" - + "}") + "The Cloud Storage path to the JSON file that defines your BigQuery schema.") ValueProvider getSchemaJSONPath(); void setSchemaJSONPath(ValueProvider value); @@ -145,8 +122,8 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { order = 3, description = "BigQuery output table", helpText = - "BigQuery table location to write the output to. The table's schema must match the " - + "input objects.") + "The name of the BigQuery table that stores your processed data. If you reuse an existing " + + "BigQuery table, the data is appended to the destination table.") ValueProvider getOutputTable(); void setOutputTable(ValueProvider value); @@ -154,7 +131,7 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { @TemplateParameter.GcsWriteFolder( order = 4, description = "Temporary directory for BigQuery loading process", - helpText = "Temporary directory for BigQuery loading process", + helpText = "The temporary directory to use during the BigQuery loading process.", example = "gs://your-bucket/your-files/temp_dir") @Validation.Required ValueProvider getBigQueryLoadingTemporaryDirectory(); @@ -165,7 +142,9 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { order = 5, description = "BigQuery output table for bad records", helpText = - "BigQuery table location to write the bad record. The table's schema must match the {RawContent: STRING, ErrorMsg:STRING}") + "The name of the BigQuery table to use to store the rejected data when processing the" + + " CSV files. If you reuse an existing BigQuery table, the data is appended to the" + + " destination table. The schema of this table must match the" + "error table schema (https://cloud.google.com/dataflow/docs/guides/templates/provided/cloud-storage-csv-to-bigquery#GcsCSVToBigQueryBadRecordsSchema).") ValueProvider getBadRecordsOutputTable(); void setBadRecordsOutputTable(ValueProvider value); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java b/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java index 1a66a4166d..4f62eaa53d 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java @@ -108,8 +108,7 @@ public interface Options order = 2, description = "Pub/Sub input topic", helpText = - "Pub/Sub topic to read the input from, in the format of " - + "'projects/your-project-id/topics/your-topic-name'") + "Pub/Sub topic to subscribe for message consumption. The topic name must be in the format of `projects//topics/`.") ValueProvider getInputTopic(); void setInputTopic(ValueProvider value); @@ -126,8 +125,7 @@ public interface Options order = 4, description = "Output file directory in Cloud Storage", helpText = - "The path and filename prefix for writing output files. Must end with a slash. DateTime" - + " formatting is used to parse directory path for date & time formatters.") + "Output directory where output Avro files are archived. Must contain / at the end. For example: `gs://example-bucket/example-directory/`") @Required ValueProvider getOutputDirectory(); @@ -137,7 +135,7 @@ public interface Options order = 5, optional = true, description = "Output filename prefix of the files to write", - helpText = "The prefix to place on each windowed file.", + helpText = "Output filename prefix for the Avro files.", regexes = "^[a-zA-Z\\-]+$") @Default.String("output") ValueProvider getOutputFilenamePrefix(); @@ -149,8 +147,7 @@ public interface Options optional = true, description = "Output filename suffix of the files to write", helpText = - "The suffix to place on each windowed file. Typically a file extension such " - + "as .txt or .csv.") + "Output filename suffix for the Avro files.") @Default.String("") ValueProvider getOutputFilenameSuffix(); @@ -159,7 +156,7 @@ public interface Options @TemplateParameter.GcsWriteFolder( order = 7, description = "Temporary Avro write directory", - helpText = "Directory for temporary Avro files.") + helpText = "Directory for temporary Avro files. Must contain / at the end. For example: `gs://example-bucket/example-directory/`.") @Required ValueProvider getAvroTempDirectory(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java b/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java index 475ec9f88e..193490edb8 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java @@ -88,7 +88,7 @@ public interface SpannerToTextOptions order = 1, optional = true, description = "Cloud Storage temp directory for storing CSV files", - helpText = "The Cloud Storage path where the temporary CSV files can be stored.", + helpText = "The Cloud Storage path where temporary CSV files are written.", example = "gs://your-bucket/your-path") ValueProvider getCsvTempDirectory(); @@ -105,8 +105,7 @@ public interface SpannerToTextOptions optional = true, description = "Priority for Spanner RPC invocations", helpText = - "The request priority for Cloud Spanner calls. The value must be one of:" - + " [HIGH,MEDIUM,LOW].") + "The" + "request priority (https://cloud.google.com/spanner/docs/reference/rest/v1/RequestOptions)" + "for Spanner calls. Possible values are `HIGH`, `MEDIUM`, `LOW`. The default value is `MEDIUM`.") ValueProvider getSpannerPriority(); void setSpannerPriority(ValueProvider value); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java index d5bbb2e172..87cc1e81a3 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java @@ -75,7 +75,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 1, optional = true, description = "Whether input CSV files contain a header record.", - helpText = "Input CSV files contain a header record (true/false).") + helpText = "Boolean to denote whether headers are included in the CSV. Default `false`.") @Default.Boolean(false) ValueProvider getContainsHeaders(); @@ -85,7 +85,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 2, description = "Column delimiter of the data files.", helpText = - "The column delimiter of the input text files. Default: use delimiter provided in csvFormat", + "The delimiter that the CSV uses.", example = ",") ValueProvider getDelimiter(); @@ -95,8 +95,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 3, description = "CSV Format to use for parsing records.", helpText = - "CSV format specification to use for parsing records. See https://commons.apache.org/proper/commons-csv/apidocs/org/apache/commons/csv/CSVFormat.html for more details. Must match format names exactly found at: " - + "https://commons.apache.org/proper/commons-csv/apidocs/org/apache/commons/csv/CSVFormat.Predefined.html") + "The CSV format according to Apache Commons CSV format. Default: `Default.`") ValueProvider getCsvFormat(); void setCsvFormat(ValueProvider csvFormat); @@ -107,8 +106,7 @@ public interface CsvPipelineOptions extends PipelineOptions { regexes = {"^(US-ASCII|ISO-8859-1|UTF-8|UTF-16)$"}, description = "CSV file encoding", helpText = - "CSV file character encoding format. Allowed Values are US-ASCII" - + ", ISO-8859-1, UTF-8, UTF-16") + "CSV file encoding.") @Default.String("UTF-8") ValueProvider getCsvFileEncoding(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java index 9336bd972b..0a1159922c 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java @@ -85,7 +85,7 @@ public interface SpannerReadOptions extends PipelineOptions { order = 1, regexes = {"^.+$"}, description = "Spanner Table", - helpText = "Spanner Table to read from") + helpText = "The table to read the data from.") ValueProvider getSpannerTable(); @SuppressWarnings("unused") @@ -95,7 +95,7 @@ public interface SpannerReadOptions extends PipelineOptions { order = 2, description = "Read data from Cloud Spanner Project Id", helpText = - "The Google Cloud Project Id of the Cloud Spanner database that you want to read data from") + "The Google Cloud Project ID of the Spanner database that you want to read data from.") ValueProvider getSpannerProjectId(); @SuppressWarnings("unused") @@ -105,7 +105,7 @@ public interface SpannerReadOptions extends PipelineOptions { order = 3, regexes = {".+"}, description = "Read data from Cloud Spanner Instance", - helpText = "Instance of requested table.") + helpText = "The instance ID of the requested table.") ValueProvider getSpannerInstanceId(); @SuppressWarnings("unused") @@ -115,7 +115,7 @@ public interface SpannerReadOptions extends PipelineOptions { order = 4, regexes = {".+"}, description = "Read data from Cloud Spanner Database ", - helpText = "Database of requested table.") + helpText = "The database ID of the requested table.") ValueProvider getSpannerDatabaseId(); @SuppressWarnings("unused") @@ -141,10 +141,9 @@ public interface SpannerReadOptions extends PipelineOptions { }, description = "Snapshot time", helpText = - "If set, specifies the time when the snapshot must be taken." - + " String is in the RFC 3339 format in UTC time. " - + " Timestamp must be in the past and Maximum timestamp staleness applies." - + "https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness", + "The timestamp that corresponds to the version of the Spanner database that you want to read." + + " The timestamp must be specified as per" + "RFC 3339 (https://tools.ietf.org/html/rfc3339)" + "UTC \"Zulu\" format. " + + " The timestamp must be in the past and" + "Maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness)" + "applies.", example = "1990-12-31T23:59:60Z") @Default.String(value = "") ValueProvider getSpannerSnapshotTime(); @@ -157,9 +156,9 @@ public interface SpannerReadOptions extends PipelineOptions { optional = true, description = "Use independent compute resource (Spanner DataBoost).", helpText = - "Use Spanner on-demand compute so the export job will run on independent compute" - + " resources and have no impact to current Spanner workloads. This will incur" - + " additional charges in Spanner.") + "Set to `true` to use the compute resources of Spanner Data Boost to run the job with near-zero" + + " impact on Spanner OLTP workflows. This requires the `spanner.databases.useDataBoost` Identity and" + + " Access Management (IAM) permission. For more information, see" + "Data Boost overview (https://cloud.google.com/spanner/docs/databoost/databoost-overview).") @Default.Boolean(false) ValueProvider getDataBoostEnabled(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/TextConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/TextConverters.java index b4e0181ace..2915fce690 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/TextConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/TextConverters.java @@ -43,7 +43,7 @@ public interface FilesystemWriteOptions extends PipelineOptions { @TemplateParameter.GcsWriteFolder( order = 2, description = "Output file directory in Cloud Storage", - helpText = "The path and filename prefix for writing output files.", + helpText = "The Cloud Storage path where output text files are written. Add / at the end.", example = "gs://your-bucket/your-path") ValueProvider getTextWritePrefix(); diff --git a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java index 9c59a59022..13c5188c1c 100644 --- a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java +++ b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java @@ -31,7 +31,7 @@ public interface BigtableCommonOptions extends GcpOptions { order = 1, optional = true, description = "The timeout for an RPC attempt in milliseconds", - helpText = "This sets the timeout for an RPC attempt in milliseconds") + helpText = "The timeout for each Bigtable RPC attempt in milliseconds.") Integer getBigtableRpcAttemptTimeoutMs(); void setBigtableRpcAttemptTimeoutMs(Integer value); @@ -40,7 +40,7 @@ public interface BigtableCommonOptions extends GcpOptions { order = 2, optional = true, description = "The total timeout for an RPC operation in milliseconds", - helpText = "This sets the total timeout for an RPC operation in milliseconds") + helpText = "The total timeout for a Bigtable RPC operation in milliseconds.") Integer getBigtableRpcTimeoutMs(); void setBigtableRpcTimeoutMs(Integer value); @@ -49,7 +49,7 @@ public interface BigtableCommonOptions extends GcpOptions { order = 3, optional = true, description = "The additional retry codes", - helpText = "This sets the additional retry codes, separated by ','", + helpText = "The additional retry codes.", example = "RESOURCE_EXHAUSTED,DEADLINE_EXCEEDED") String getBigtableAdditionalRetryCodes(); @@ -62,7 +62,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 1, regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Bigtable Instance ID", - helpText = "The ID of the Cloud Bigtable instance that contains the table") + helpText = "The ID of the Bigtable instance that contains the table.") @Validation.Required String getBigtableWriteInstanceId(); @@ -72,7 +72,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 2, regexes = {"[_a-zA-Z0-9][-_.a-zA-Z0-9]*"}, description = "Bigtable Table ID", - helpText = "The ID of the Cloud Bigtable table to write") + helpText = "The ID of the Bigtable table to write to.") @Validation.Required String getBigtableWriteTableId(); @@ -82,7 +82,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 3, regexes = {"[-_.a-zA-Z0-9]+"}, description = "The Bigtable Column Family", - helpText = "This specifies the column family to write data into") + helpText = "The name of the column family of the Bigtable table to write data into.") @Validation.Required String getBigtableWriteColumnFamily(); @@ -94,8 +94,8 @@ interface WriteOptions extends BigtableCommonOptions { regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Bigtable App Profile", helpText = - "Bigtable App Profile to use for the export. The default for this parameter " - + "is the Bigtable instance's default app profile") + "The ID of the Bigtable application profile to be used for the export. If you" + + "do not specify an app profile, Bigtable uses the" + "default app profile (https://cloud.google.com/bigtable/docs/app-profiles#default-app-profile)" + "of the instance.") @Default.String("default") String getBigtableWriteAppProfile(); @@ -106,8 +106,8 @@ interface WriteOptions extends BigtableCommonOptions { optional = true, description = "Bigtable Project ID", helpText = - "The ID of the Google Cloud project of the Cloud Bigtable instance that you want " - + "to write data to.") + "The ID of the Google Cloud project of the Bigtable instance that you want" + + "to write data to.") String getBigtableWriteProjectId(); void setBigtableWriteProjectId(String value); @@ -116,7 +116,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 6, optional = true, description = "Bigtable's latency target in milliseconds for latency-based throttling", - helpText = "This enables latency-based throttling and specifies the target latency") + helpText = "The latency target of Bigtable in milliseconds for latency-based throttling.") Integer getBigtableBulkWriteLatencyTargetMs(); void setBigtableBulkWriteLatencyTargetMs(Integer value); @@ -125,7 +125,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 7, optional = true, description = "The max number of row keys in a Bigtable batch write operation", - helpText = "This sets the max number of row keys in a Bigtable batch write operation") + helpText = "The max number of row keys in a Bigtable batch write operation.") Integer getBigtableBulkWriteMaxRowKeyCount(); void setBigtableBulkWriteMaxRowKeyCount(Integer value); @@ -134,7 +134,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 8, optional = true, description = "The max amount of bytes in a Bigtable batch write operation", - helpText = "This sets the max amount of bytes in a Bigtable batch write operation") + helpText = "The max amount of bytes to include per Bigtable batch write operation.") Integer getBigtableBulkWriteMaxRequestSizeBytes(); void setBigtableBulkWriteMaxRequestSizeBytes(Integer value); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java index 6f1a11d970..dc58a3c787 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryCommonOptions.java @@ -37,8 +37,7 @@ public interface WriteOptions extends PipelineOptions { order = 1, description = "BigQuery output table", helpText = - "BigQuery table location to write the output to. The name should be in the format " - + "`:.`. The table's schema must match input objects.") + "The BigQuery output table location to write the Apache Kafka messages to, in the format of `my-project:dataset.table") @Required String getOutputTableSpec(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryStorageApiStreamingOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryStorageApiStreamingOptions.java index c919e8ceee..a787beddc5 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryStorageApiStreamingOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/BigQueryStorageApiStreamingOptions.java @@ -26,13 +26,8 @@ public interface BigQueryStorageApiStreamingOptions extends BigQueryOptions { optional = true, description = "Use BigQuery Storage Write API", helpText = - "If true, the pipeline uses the Storage Write API when writing the data to BigQuery (see" - + " https://cloud.google.com/blog/products/data-analytics/streaming-data-into-bigquery-using-storage-write-api)." - + " The default value is false. When using Storage Write API in exactly-once mode," - + " you must set the following parameters: \"Number of streams for BigQuery Storage Write API\"" - + " and \"Triggering frequency in seconds for BigQuery Storage Write API\"." - + " If you enable Dataflow at-least-once mode or set the useStorageWriteApiAtLeastOnce parameter" - + " to true, then you don't need to set the number of streams or the triggering frequency.") + "If `true`, the pipeline uses the BigQuery Storage Write API (https://cloud.google.com/bigquery/docs/write-api). The default" + + " value is `false`. For more information, see Using the Storage Write API (https://beam.apache.org/documentation/io/built-in/google-bigquery/#storage-write-api)") @Default.Boolean(false) @Override Boolean getUseStorageWriteApi(); @@ -42,9 +37,11 @@ public interface BigQueryStorageApiStreamingOptions extends BigQueryOptions { optional = true, description = "Use at at-least-once semantics in BigQuery Storage Write API", helpText = - "This parameter takes effect only if \"Use BigQuery Storage Write API\" is enabled. If" - + " enabled the at-least-once semantics will be used for Storage Write API, otherwise" - + " exactly-once semantics will be used.") + "When using the Storage Write API, specifies the write" + + " semantics. To use at-least-once semantics (https://beam.apache.org/documentation/io/built-in/google-bigquery/#at-least-once-semantics), set this parameter to" + + " `true`. To use exactly-once semantics, set the parameter to `false`. This" + + " parameter applies only when `useStorageWriteApi` is `true`. The" + + " default value is `false`.") @Default.Boolean(false) @Override Boolean getUseStorageWriteApiAtLeastOnce(); @@ -54,11 +51,10 @@ public interface BigQueryStorageApiStreamingOptions extends BigQueryOptions { optional = true, description = "Number of streams for BigQuery Storage Write API", helpText = - "Number of streams defines the parallelism of the BigQueryIO’s Write transform and" - + " roughly corresponds to the number of Storage Write API’s streams which will be" - + " used by the pipeline. See" - + " https://cloud.google.com/blog/products/data-analytics/streaming-data-into-bigquery-using-storage-write-api" - + " for the recommended values.") + "When using the Storage Write API, specifies the number of write" + + " streams. If `useStorageWriteApi` is `true` and" + + " `useStorageWriteApiAtLeastOnce` is `false`, then you must set this" + + " parameter.") @Override @Default.Integer(0) Integer getNumStorageWriteApiStreams(); @@ -68,10 +64,10 @@ public interface BigQueryStorageApiStreamingOptions extends BigQueryOptions { optional = true, description = "Triggering frequency in seconds for BigQuery Storage Write API", helpText = - "Triggering frequency will determine how soon the data will be visible for querying in" - + " BigQuery. See" - + " https://cloud.google.com/blog/products/data-analytics/streaming-data-into-bigquery-using-storage-write-api" - + " for the recommended values.") + "When using the Storage Write API, specifies the triggering frequency," + + " in seconds. If `useStorageWriteApi` is `true` and" + + " `useStorageWriteApiAtLeastOnce` is `false`, then you must set this" + + " parameter.") @Override Integer getStorageWriteApiTriggeringFrequencySec(); } diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/CommonTemplateOptions.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/CommonTemplateOptions.java index 7c73746d16..75c7d93260 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/options/CommonTemplateOptions.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/options/CommonTemplateOptions.java @@ -42,8 +42,7 @@ public interface CommonTemplateOptions extends PipelineOptions { }, description = "Extra files to stage in the workers", helpText = - "Comma separated Cloud Storage paths or Secret Manager secrets for files to stage " - + "in the worker. These files will be saved under the `/extra_files` directory in each worker", + "Comma separated Cloud Storage paths or Secret Manager secrets for files to stage in the worker. These files will be saved under the `/extra_files` directory in each worker. ", example = "gs://your-bucket/file.txt,projects/project-id/secrets/secret-id/versions/version-id") String getExtraFilesToStage(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index 1d9d7d738a..f238ca09f9 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -166,7 +166,9 @@ public interface BigQueryReadOptions extends PipelineOptions { order = 1, optional = true, description = "BigQuery source table", - helpText = "BigQuery source table spec.", + helpText = "The BigQuery table to read from. Format: `projectId:datasetId.tablename`. If you specify `inputTableSpec`, the template reads the data directly from BigQuery storage by using the" + "BigQuery Storage Read API. (https://cloud.google.com/bigquery/docs/reference/storage)" + "For information about limitations in the Storage Read API, see" + + "Limitations.(https://cloud.google.com/bigquery/docs/reference/storage#limitations)" + + "You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "bigquery-project:dataset.input_table") String getInputTableSpec(); @@ -177,9 +179,10 @@ public interface BigQueryReadOptions extends PipelineOptions { optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "Messages failed to reach the output table for all kind of reasons (e.g., mismatched " - + "schema, malformed json) are written to this table. If it doesn't exist, it will be " - + "created during pipeline execution.", + "The BigQuery table for messages that failed to reach the output" + + "table, in the format of `my-project:dataset.my-deadletter-table` " + + "If it doesn't exist, the table is created during pipeline execution. If" + + "not specified, `_error_records` is used instead.", example = "your-project-id:your-dataset.your-table-name") String getOutputDeadletterTable(); @@ -190,7 +193,9 @@ public interface BigQueryReadOptions extends PipelineOptions { optional = true, regexes = {"^.+$"}, description = "Input SQL query.", - helpText = "Query to be executed on the source to extract the data.", + helpText = "The SQL query to use to read data from BigQuery. If the BigQuery dataset is in a different project than the Dataflow job, specify the full dataset name in the SQL query, as follows: `projectId.datasetName.tablename`. By default, the `query` parameter uses" + + "GoogleSQL (https://cloud.google.com/bigquery/docs/introduction-sql)" + ", unless `useLegacySql` is `true`." + + "You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "select * from sampledb.sample_table") String getQuery(); @@ -200,7 +205,8 @@ public interface BigQueryReadOptions extends PipelineOptions { order = 4, optional = true, description = "Set to true to use legacy SQL", - helpText = "Set to true to use legacy SQL (only applicable if supplying query).") + helpText = "Set to `true` to use legacy SQL. This parameter only applies when using" + + "the `query` parameter. Default: `false`.") @Default.Boolean(false) Boolean getUseLegacySql(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java index 935550839e..94c04f3a78 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java @@ -183,7 +183,7 @@ public interface CsvPipelineOptions extends PipelineOptions { @TemplateParameter.GcsReadFile( order = 1, description = "The input filepattern to read from.", - helpText = "Cloud storage file pattern glob to read from. ex: gs://your-bucket/path/*.csv") + helpText = "The Cloud Storage file pattern to search for CSV files. Example: `gs://mybucket/test-*.csv.`") String getInputFileSpec(); void setInputFileSpec(String inputFileSpec); @@ -237,7 +237,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 6, optional = true, description = "Path to JSON schema", - helpText = "Path to JSON schema. Default: null.", + helpText = "The Path to JSON schema. Default: `null`.", example = "gs://path/to/schema") String getJsonSchemaPath(); @@ -247,7 +247,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 7, optional = true, description = "Set to true if number of files is in the tens of thousands", - helpText = "Set to true if number of files is in the tens of thousands.") + helpText = "Set to true if number of files is in the tens of thousands. Default: `false`") @Default.Boolean(false) Boolean getLargeNumFiles(); @@ -259,8 +259,7 @@ public interface CsvPipelineOptions extends PipelineOptions { regexes = {"^(US-ASCII|ISO-8859-1|UTF-8|UTF-16)$"}, description = "CSV file encoding", helpText = - "CSV file character encoding format. Allowed Values are US-ASCII" - + ", ISO-8859-1, UTF-8, UTF-16") + "CSV file encoding.") @Default.String("UTF-8") String getCsvFileEncoding(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java index 3e47ae19b5..1d3e2cba18 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java @@ -78,8 +78,7 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to Javascript UDF source", helpText = - "The Cloud Storage path pattern for the JavaScript code containing your user-defined " - + "functions.", + "The Cloud Storage path pattern for the JavaScript code that contains your user-defined functions. For example: `gs://your-bucket/your-function.js`.", example = "gs://your-bucket/your-function.js") String getJavascriptTextTransformGcsPath(); @@ -91,7 +90,7 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { regexes = {"[a-zA-Z0-9_]+"}, description = "UDF Javascript Function Name", helpText = - "The name of the function to call from your JavaScript file. Use only letters, digits, and underscores.", + "The name of the function to call from your JavaScript file. Use only letters, digits, and underscores. For example: `transform` or `transform_udf1`.", example = "'transform' or 'transform_udf1'") String getJavascriptTextTransformFunctionName(); @@ -102,7 +101,7 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "JavaScript UDF auto-reload interval (minutes)", helpText = - "Define the interval that workers may check for JavaScript UDF changes to reload the files.") + "Specifies how frequently to reload the UDF, in minutes. If the value is greater than 0, Dataflow periodically checks the UDF file in Cloud Storage, and reloads the UDF if the file is modified. This parameter allows you to update the UDF while the pipeline is running, without needing to restart the job. If the value is 0, UDF reloading is disabled. The default value is 0.") @Default.Integer(0) Integer getJavascriptTextTransformReloadIntervalMinutes(); diff --git a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java index 37c1ad27c3..dc2e271cdf 100644 --- a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java +++ b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java @@ -29,7 +29,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 1, description = "Elasticsearch URL or CloudID if using Elastic Cloud", helpText = - "Elasticsearch URL in the format 'https://hostname:[port]' or specify CloudID if using Elastic Cloud", + "Elasticsearch URL in the format `https://hostname:[port]` or specify CloudID if using Elastic Cloud.", example = "https://elasticsearch-host:9200") @Validation.Required String getConnectionUrl(); @@ -40,7 +40,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 2, description = "Base64 Encoded API Key for access without requiring basic authentication", helpText = - "Base64 Encoded API key used for authentication. Refer to: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html#security-api-create-api-key-request") + "Base64 Encoded API key used for authentication.") @Validation.Required String getApiKey(); @@ -71,7 +71,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "Elasticsearch index", - helpText = "The index toward which the requests will be issued", + helpText = "The Elasticsearch index toward which the requests will be issued such as `my-index.`", example = "my-index") String getIndex(); @@ -81,7 +81,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 6, optional = true, description = "Batch Size", - helpText = "Batch size in number of documents. Default: '1000'.") + helpText = "Batch size in number of documents. Default: `1000`") @Default.Long(1000) Long getBatchSize(); @@ -92,7 +92,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Batch Size in Bytes", helpText = - "Batch Size in bytes used for batch insertion of messages into elasticsearch. Default: '5242880 (5mb)'") + "Batch size in number of bytes. Default: `5242880` (5mb).") @Default.Long(5242880) Long getBatchSizeBytes(); @@ -102,7 +102,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 8, optional = true, description = "Max retry attempts.", - helpText = "Max retry attempts, must be > 0. Default: 'no retries'") + helpText = "Max retry attempts, must be > 0. Default: no retries.") Integer getMaxRetryAttempts(); void setMaxRetryAttempts(Integer maxRetryAttempts); @@ -111,7 +111,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 9, optional = true, description = "Max retry duration.", - helpText = "Max retry duration in milliseconds, must be > 0. Default: 'no retries'") + helpText = "Max retry duration in milliseconds, must be > 0. Default: no retries.") Long getMaxRetryDuration(); void setMaxRetryDuration(Long maxRetryDuration); @@ -121,7 +121,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Document property to specify _index metadata", helpText = - "A property in the document being indexed whose value will specify '_index' metadata to be included with document in bulk request (takes precedence over an '_index' UDF). Default: none") + "A property in the document being indexed whose value will specify `_index` metadata to be included with document in bulk request (takes precedence over an `_index` UDF). Default: none.") String getPropertyAsIndex(); void setPropertyAsIndex(String propertyAsIndex); @@ -131,7 +131,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for _index metadata", helpText = - "The Cloud Storage path to the JavaScript UDF source for a function that will specify '_index' metadata to be included with document in bulk request. Default: none") + "The Cloud Storage path to the JavaScript UDF source for a function that will specify `_index` metadata to be included with document in bulk request. Default: none.") String getJavaScriptIndexFnGcsPath(); void setJavaScriptIndexFnGcsPath(String javaScriptTextTransformGcsPath); @@ -141,7 +141,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for _index metadata", helpText = - "UDF JavaScript function Name for function that will specify _index metadata to be included with document in bulk request. Default: none") + "UDF JavaScript function name for function that will specify `_index` metadata to be included with document in bulk request. Default: none.") String getJavaScriptIndexFnName(); void setJavaScriptIndexFnName(String javaScriptTextTransformFunctionName); @@ -151,7 +151,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Document property to specify _id metadata", helpText = - "A property in the document being indexed whose value will specify '_id' metadata to be included with document in bulk request (takes precedence over an '_id' UDF). Default: none") + "A property in the document being indexed whose value will specify `_id` metadata to be included with document in bulk request (takes precedence over an `_id` UDF). Default: none.") String getPropertyAsId(); void setPropertyAsId(String propertyAsId); @@ -161,7 +161,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for _id metadata", helpText = - "The Cloud Storage path to the JavaScript UDF source for a function that will specify '_id' metadata to be included with document in bulk request.Default: none") + "The Cloud Storage path to the JavaScript UDF source for a function that will specify `_id` metadata to be included with document in bulk request. Default: none.") String getJavaScriptIdFnGcsPath(); void setJavaScriptIdFnGcsPath(String javaScriptTextTransformGcsPath); @@ -171,7 +171,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for _id metadata", helpText = - "UDF JavaScript Function Name for function that will specify _id metadata to be included with document in bulk request. Default: none") + "UDF JavaScript function name for function that will specify `_id` metadata to be included with document in bulk request. Default: none.") String getJavaScriptIdFnName(); void setJavaScriptIdFnName(String javaScriptTextTransformFunctionName); @@ -181,7 +181,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for _type metadata", helpText = - "The Cloud Storage path to the JavaScript UDF source for function that will specify '_type' metadata to be included with document in bulk request. Default: none") + "The Cloud Storage path to the JavaScript UDF source for a function that will specify `_type` metadata to be included with document in bulk request. Default: none.") String getJavaScriptTypeFnGcsPath(); void setJavaScriptTypeFnGcsPath(String javaScriptTextTransformGcsPath); @@ -191,7 +191,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for _type metadata", helpText = - "UDF JavaScript function Name for function that will specify '_type' metadata to be included with document in bulk request. Default: none") + "UDF JavaScript function name for function that will specify `_type` metadata to be included with document in bulk request. Default: none.") String getJavaScriptTypeFnName(); void setJavaScriptTypeFnName(String javaScriptTextTransformFunctionName); @@ -201,7 +201,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for isDelete function", helpText = - "The Cloud Storage path to JavaScript UDF source for function that will determine if document should be deleted rather than inserted or updated. The function should return string value \"true\" or \"false\". Default: none") + "The Cloud Storage path to JavaScript UDF source for function that will determine if document should be deleted rather than inserted or updated. The function should return string value `true` or `false`. Default: none.") String getJavaScriptIsDeleteFnGcsPath(); void setJavaScriptIsDeleteFnGcsPath(String javaScriptTextTransformGcsPath); @@ -211,7 +211,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for isDelete", helpText = - "UDF JavaScript function Name for function that will determine if document should be deleted rather than inserted or updated. The function should return string value \"true\" or \"false\". Default: none") + "UDF JavaScript function name for function that will determine if document should be deleted rather than inserted or updated. The function should return string value `true` or `false`. Default: none.") String getJavaScriptIsDeleteFnName(); void setJavaScriptIsDeleteFnName(String javaScriptTextTransformFunctionName); @@ -221,7 +221,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Use partial updates", helpText = - "Whether to use partial updates (update rather than create or index, allowing partial docs) with Elasticsearch requests. Default: 'false'") + "Whether to use partial updates (update rather than create or index, allowing partial docs) with Elasticsearch requests. Default: `false`.") @Default.Boolean(false) Boolean getUsePartialUpdate(); @@ -233,7 +233,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Build insert method", helpText = - "Whether to use 'INDEX' (index, allows upsert) or 'CREATE' (create, errors on duplicate _id) with Elasticsearch bulk requests. Default: 'CREATE'") + "Whether to use `INDEX` (index, allows upserts) or `CREATE` (create, errors on duplicate _id) with Elasticsearch bulk requests. Default: `CREATE`.") @Default.Enum("CREATE") BulkInsertMethodOptions getBulkInsertMethod(); diff --git a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java index 1efaab0d45..6ddb6fd2f7 100644 --- a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java +++ b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java @@ -34,7 +34,7 @@ public interface GCSToElasticsearchOptions order = 1, description = "BigQuery Deadletter table to send failed inserts.", helpText = - "Messages failed to reach the target for all kind of reasons (e.g., mismatched schema, malformed json) are written to this table.", + "The BigQuery Deadletter table to send failed inserts to.", example = "your-project:your-dataset.your-table-name") @Validation.Required String getDeadletterTable(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java index b6d9809388..fd6ed6cb3e 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java @@ -33,8 +33,8 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Spanner Project ID", helpText = - "Project to read change streams from. The default for this parameter is the project " - + "where the Dataflow pipeline is running.") + "The project to read change streams from. This is also the project where the change " + + "streams connector metadata table is created. The default for this parameter is the project where the Dataflow pipeline is running.") @Default.String("") String getSpannerProjectId(); @@ -63,10 +63,10 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Spanner database role", helpText = - "Database role user assumes while reading from the change stream. The database role" - + " should have required privileges to read from change stream. If a database role is" - + " not specified, the user should have required IAM permissions to read from the" - + " database.") + "The Spanner database role to use when running the template. This parameter is required" + + " only when the IAM principal who is running the template is a fine-grained access control user. The" + + " database role must have the `SELECT` privilege on the change stream and the `EXECUTE` privilege on" + + " the change stream's read function. For more information, see Fine-grained access control for change streams (https://cloud.google.com/spanner/docs/fgac-change-streams).") String getSpannerDatabaseRole(); void setSpannerDatabaseRole(String spannerDatabaseRole); @@ -84,9 +84,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt order = 6, description = "Spanner metadata database", helpText = - "The Spanner database to use for the change streams connector metadata table. For change" - + " streams tracking all tables in a database, we recommend putting the metadata" - + " table in a separate database.") + "TThe Spanner database to use for the change streams connector metadata table.") @Validation.Required String getSpannerMetadataDatabase(); @@ -97,10 +95,10 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Cloud Spanner metadata table name", helpText = - "The Cloud Spanner change streams connector metadata table name to use. If not provided," - + " a Cloud Spanner change streams connector metadata table will automatically be" - + " created during the pipeline flow. This parameter must be provided when updating" - + " an existing pipeline and should not be provided otherwise.") + "The Spanner change streams connector metadata table name to use. If not provided," + + " Spanner automatically creates the streams connector metadata table during the pipeline flow" + + " change. You must provide this parameter when updating an existing pipeline. Don't use this" + + " parameter for other cases.") String getSpannerMetadataTableName(); void setSpannerMetadataTableName(String value); @@ -119,9 +117,9 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "The timestamp to read change streams from", helpText = - "The starting DateTime, inclusive, to use for reading change streams" - + " (https://tools.ietf.org/html/rfc3339). For example, 2022-05-05T07:59:59Z." - + " Defaults to the timestamp when the pipeline starts.") + "The starting DateTime (https://tools.ietf.org/html/rfc3339), inclusive, to use for reading change streams. For example, ex-" + + " 2021-10-12T07:20:50.52Z. Defaults to the timestamp when the pipeline starts, that is, the current" + + " time.") @Default.String("") String getStartTimestamp(); @@ -132,9 +130,8 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "The timestamp to read change streams to", helpText = - "The ending DateTime, inclusive, to use for reading change streams" - + " (https://tools.ietf.org/html/rfc3339). Ex-2022-05-05T07:59:59Z. Defaults to an" - + " infinite time in the future.") + "The ending DateTime (https://tools.ietf.org/html/rfc3339), inclusive, to use for reading change streams. For example, ex-" + + " 2021-10-12T07:20:50.52Z. Defaults to an infinite time in the future.") @Default.String("") String getEndTimestamp(); @@ -156,7 +153,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Output data format", helpText = - "The format of the output to Pub/Sub. Allowed formats are JSON, AVRO. Default is JSON.") + "The format of the output. Output is wrapped in many PubsubMessages and sent to a Pub/Sub topic. Allowed formats are JSON and AVRO. Default is JSON.") @Default.String("JSON") String getOutputDataFormat(); @@ -167,9 +164,9 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Pub/Sub API", helpText = - "Pub/Sub API used to implement the pipeline. Allowed APIs are pubsubio and native_client." - + " Default is pubsubio. For a small QPS, native_client can achieve a smaller latency" - + " than pubsubio. For a large QPS, pubsubio has better and more stable performance.") + "Pub/Sub API used to implement the pipeline. Allowed APIs are `pubsubio` and `native_client`." + + " For a small number of queries per second (QPS), `native_client` has less latency. For a" + + " large number of QPS, `pubsubio` provides better and more stable performance. The default is `pubsubio`.") @Default.String("pubsubio") String getPubsubAPI(); @@ -190,7 +187,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.Text( order = 15, description = "The output Pub/Sub topic", - helpText = "The Pub/Sub topic to publish PubsubMessage.") + helpText = "The Pub/Sub topic for change streams output.") @Validation.Required String getPubsubTopic(); @@ -206,8 +203,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Priority for Spanner RPC invocations", helpText = - "The request priority for Cloud Spanner calls. The value must be one of:" - + " [HIGH,MEDIUM,LOW].") + "The request priority for Spanner calls. The value must be one of: [HIGH,MEDIUM,LOW]. (Default: HIGH)") @Default.Enum("HIGH") RpcPriority getRpcPriority(); diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java index 98514b5cfa..029f67e710 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java @@ -28,7 +28,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = false, regexes = {"^.+$"}, description = "JDBC driver class name.", - helpText = "JDBC driver class name to use.", + helpText = "The JDBC driver class name.", example = "com.mysql.jdbc.Driver") String getDriverClassName(); @@ -42,8 +42,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { }, description = "JDBC connection URL string.", helpText = - "Url connection string to connect to the JDBC source. Connection string can " - + "be passed in as plaintext or as a base64 encoded string encrypted by Google Cloud KMS.", + "The JDBC connection URL string. You can pass in this value as a string that's encrypted with a Cloud KMS key and then Base64-encoded. Remove whitespace characters from the Base64-encoded string. ", example = "jdbc:mysql://some-host:3306/sampledb") String getConnectionUrl(); @@ -55,8 +54,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { regexes = {"^.+$"}, description = "JDBC connection username.", helpText = - "User name to be used for the JDBC connection. User name can be passed in as plaintext " - + "or as a base64 encoded string encrypted by Google Cloud KMS.") + "The username to be used for the JDBC connection. You can pass in this value encrypted by a Cloud KMS key as a Base64-encoded string. For example, `echo -n 'some_username' | glcloud kms encrypt --location=my_location --keyring=mykeyring --key=mykey --plaintext-file=- --ciphertext-file=- | base64`") String getUsername(); void setUsername(String username); @@ -66,8 +64,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = true, description = "JDBC connection password.", helpText = - "Password to be used for the JDBC connection. Password can be passed in as plaintext " - + "or as a base64 encoded string encrypted by Google Cloud KMS.") + "The password to be used for the JDBC connection. You can pass in this value encrypted by a Cloud KMS key as a Base64-encoded string. For example, `echo -n 'some_password' | glcloud kms encrypt --location=my_location --keyring=mykeyring --key=mykey --plaintext-file=- --ciphertext-file=- | base64`") String getPassword(); void setPassword(String password); @@ -77,7 +74,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = false, regexes = {"^.+$"}, description = "Cloud Storage paths for JDBC drivers", - helpText = "Comma separate Cloud Storage paths for JDBC drivers.", + helpText = "Comma separated Cloud Storage paths for JDBC drivers.", example = "gs://your-bucket/driver_jar1.jar,gs://your-bucket/driver_jar2.jar") String getDriverJars(); @@ -89,7 +86,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, description = "JDBC connection property string.", helpText = - "Properties string to use for the JDBC connection. Format of the string must be [propertyName=property;]*.", + "Properties string to use for the JDBC connection. Format of the string must be `[propertyName=property;]*`. ", example = "unicode=true;characterEncoding=UTF-8") String getConnectionProperties(); @@ -100,7 +97,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = false, regexes = {"^.+$"}, description = "JDBC source SQL query.", - helpText = "Query to be executed on the source to extract the data.", + helpText = "The query to be run on the source to extract the data.", example = "select * from sampledb.sample_table") String getQuery(); @@ -110,7 +107,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { order = 8, description = "Output Pub/Sub topic", helpText = - "The name of the topic to which data should published, in the format of 'projects/your-project-id/topics/your-topic-name'", + "The Pub/Sub topic to publish to, in the format of `projects//topics/`.", example = "projects/your-project-id/topics/your-topic-name") @Validation.Required String getOutputTopic(); @@ -122,7 +119,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = true, description = "Google Cloud KMS key", helpText = - "If this parameter is provided, password, user name and connection string should all be passed in encrypted. Encrypt parameters using the KMS API encrypt endpoint. See: https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys/encrypt", + "Cloud KMS Encryption Key to decrypt the username, password, and connection string. If Cloud KMS key is passed in, the username, password, and connection string must all be passed in encrypted and base64 encoded.", example = "projects/your-project/locations/global/keyRings/your-keyring/cryptoKeys/your-key") String getKMSEncryptionKey(); diff --git a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java index f6cbc48cf2..877dabaaa0 100644 --- a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java +++ b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java @@ -174,7 +174,7 @@ public interface KafkaToBQOptions optional = true, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", - helpText = "Kafka Bootstrap Server list, separated by commas.", + helpText = "The host address of the running Apache Kafka broker servers in a comma-separated list, each host address in the format of `35.70.252.199:9092`", example = "localhost:9092,127.0.0.1:9093") @Deprecated String getBootstrapServers(); @@ -200,7 +200,7 @@ public interface KafkaToBQOptions optional = true, regexes = {"[,a-zA-Z0-9._-]+"}, description = "Kafka topic(s) to read the input from", - helpText = "Kafka topic(s) to read the input from.", + helpText = "The Apache Kafka input topics to read from in a comma-separated list. ", example = "topic1,topic2") String getInputTopics(); diff --git a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java index 93ce262851..2339386b76 100644 --- a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java +++ b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java @@ -60,7 +60,7 @@ public interface MongoDbOptions extends PipelineOptions, DataflowPipelineOptions enumOptions = {@TemplateEnumOption("FLATTEN"), @TemplateEnumOption("NONE")}, description = "User option", helpText = - "User option: `FLATTEN` or `NONE`. `FLATTEN` flattens the documents to the single level. `NONE` stores the whole document as a JSON string.") + "`FLATTEN` or `NONE`. `FLATTEN` flattens the documents to the first level. `NONE` stores the whole document as a JSON string.") @Default.String("NONE") String getUserOption(); @@ -86,9 +86,7 @@ public interface PubSubOptions extends PipelineOptions, DataflowPipelineOptions order = 1, description = "Pub/Sub input topic", helpText = - "Pub/Sub topic to read the input from, in the format of " - + "'projects/your-project-id/topics/your-topic-name'", - example = "projects/your-project-id/topics/your-topic-name") + "The Pub/Sub input topic to read from, in the format of `projects//topics/`.") String getInputTopic(); void setInputTopic(String inputTopic); @@ -101,7 +99,7 @@ public interface BigQueryWriteOptions extends PipelineOptions, DataflowPipelineO order = 1, description = "BigQuery output table", helpText = - "BigQuery table location to write the output to. The name should be in the format `:.`. The table's schema must match input objects.") + "BigQuery table to write to. For example, `bigquery-project:dataset.output_table`.") String getOutputTableSpec(); void setOutputTableSpec(String outputTableSpec); @@ -114,7 +112,7 @@ public interface JavascriptDocumentTransformerOptions extends PipelineOptions { optional = true, description = "JavaScript UDF path in Cloud Storage.", helpText = - "The Cloud Storage path pattern for the JavaScript code containing your user-defined functions.", + "The Cloud Storage URI of the `.js` file that defines the JavaScript user-defined function (UDF) you want to use.", example = "gs://your-bucket/your-transforms/*.js") String getJavascriptDocumentTransformGcsPath(); @@ -125,7 +123,10 @@ public interface JavascriptDocumentTransformerOptions extends PipelineOptions { optional = true, description = "The name of the JavaScript function to call as your UDF.", helpText = - "The function name should only contain letters, digits and underscores. Example: 'transform' or 'transform_udf1'.", + "The name of the JavaScript user-defined function (UDF) that you" + + "want to use. For example, if your JavaScript function code is `myTransform(`" + + "`inJson) { /*...do stuff...*/ }`, then the function name is" + + "`myTransform`. For sample JavaScript UDFs, see UDF Examples (https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples).", example = "transform") String getJavascriptDocumentTransformFunctionName(); diff --git a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/templates/MongoDbToBigQueryCdc.java b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/templates/MongoDbToBigQueryCdc.java index 1072f0754c..f8c8f11d86 100644 --- a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/templates/MongoDbToBigQueryCdc.java +++ b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/templates/MongoDbToBigQueryCdc.java @@ -93,9 +93,10 @@ public interface Options optional = true, description = "Use at at-least-once semantics in BigQuery Storage Write API", helpText = - "This parameter takes effect only if \"Use BigQuery Storage Write API\" is enabled. If" - + " enabled the at-least-once semantics will be used for Storage Write API, otherwise" - + " exactly-once semantics will be used.", + "When using the Storage Write API, specifies the write semantics. To" + + " use at-least-once semantics (https://beam.apache.org/documentation/io/built-in/google-bigquery/#at-least-once-semantics), set this parameter to `true`. To use exactly-" + + " once semantics, set the parameter to `false`. This parameter applies only when" + + " `useStorageWriteApi` is `true`. The default value is `false`.", hiddenUi = true) @Default.Boolean(false) @Override diff --git a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java index 60c106a564..04e7de83c8 100644 --- a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java +++ b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java @@ -135,8 +135,8 @@ public interface PubSubToRedisOptions order = 1, description = "Pub/Sub input subscription", helpText = - "Pub/Sub subscription to read the input from, in the format of" - + " 'projects/your-project-id/subscriptions/your-subscription-name'", + "The Pub/Sub subscription to read the input from, in the format of" + + " `'projects/my-project-id/subscriptions/my-subscription-id'`.", example = "projects/your-project-id/subscriptions/your-subscription-name") String getInputSubscription(); @@ -145,7 +145,7 @@ public interface PubSubToRedisOptions @TemplateParameter.Text( order = 2, description = "Redis DB Host", - helpText = "Redis database host.", + helpText = "The Redis database host.", example = "your.cloud.db.redislabs.com") @Default.String("127.0.0.1") @Validation.Required @@ -156,7 +156,7 @@ public interface PubSubToRedisOptions @TemplateParameter.Integer( order = 3, description = "Redis DB Port", - helpText = "Redis database port.", + helpText = "The Redis database port.", example = "12345") @Default.Integer(6379) @Validation.Required @@ -167,7 +167,7 @@ public interface PubSubToRedisOptions @TemplateParameter.Text( order = 4, description = "Redis DB Password", - helpText = "Redis database password.") + helpText = "The Redis database password. Defaults to empty.") @Default.String("") @Validation.Required String getRedisPassword(); @@ -178,7 +178,7 @@ public interface PubSubToRedisOptions order = 5, optional = true, description = "Redis ssl enabled", - helpText = "Redis database ssl parameter.") + helpText = "The Redis database SSL parameter.") @Default.Boolean(false) @UnknownKeyFor @NonNull @@ -198,7 +198,7 @@ public interface PubSubToRedisOptions }, description = "Redis sink to write", helpText = - "Supported Redis sinks are STRING_SINK, HASH_SINK, STREAMS_SINK and LOGGING_SINK", + "The Redis sink. Supported values are `STRING_SINK, HASH_SINK, STREAMS_SINK, and LOGGING_SINK`.", example = "STRING_SINK") @Default.Enum("STRING_SINK") RedisSinkType getRedisSinkType(); @@ -209,7 +209,7 @@ public interface PubSubToRedisOptions order = 7, optional = true, description = "Redis connection timeout in milliseconds", - helpText = "Redis connection timeout in milliseconds.", + helpText = "The Redis connection timeout in milliseconds. ", example = "2000") @Default.Integer(2000) int getConnectionTimeout(); @@ -221,7 +221,7 @@ public interface PubSubToRedisOptions optional = true, description = "Hash key expiration time in sec (ttl)", helpText = - "Key expiration time in sec (ttl, default for HASH_SINK is -1 i.e. no expiration)") + "The key expiration time in seconds. The `ttl` default for `HASH_SINK` is -1, which means it never expires.") @Default.Long(-1L) Long getTtl(); From e84c30abfe2d231bc339b5da98732b83657624a9 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Wed, 15 May 2024 09:45:27 +0530 Subject: [PATCH 03/16] Apply suggestions from code review Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../teleport/templates/PubsubToAvro.java | 10 +++--- .../templates/common/CsvConverters.java | 8 ++--- .../templates/common/SpannerConverters.java | 6 ++-- .../options/BigtableCommonOptions.java | 8 ++--- .../v2/transforms/BigQueryConverters.java | 12 +++---- .../teleport/v2/transforms/CsvConverters.java | 8 ++--- .../options/ElasticsearchWriteOptions.java | 36 +++++++++---------- .../options/GCSToElasticsearchOptions.java | 2 +- .../SpannerChangeStreamsToPubSubOptions.java | 8 ++--- .../v2/options/JdbcToPubsubOptions.java | 14 ++++---- .../v2/templates/KafkaToBigQuery.java | 2 +- .../options/MongoDbToBigQueryOptions.java | 2 +- .../teleport/v2/templates/PubSubToRedis.java | 2 +- 13 files changed, 59 insertions(+), 59 deletions(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java b/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java index 4f62eaa53d..54195301ea 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java @@ -108,7 +108,7 @@ public interface Options order = 2, description = "Pub/Sub input topic", helpText = - "Pub/Sub topic to subscribe for message consumption. The topic name must be in the format of `projects//topics/`.") + "The Pub/Sub topic to subscribe to for message consumption. The topic name must be in the format projects//topics/.") ValueProvider getInputTopic(); void setInputTopic(ValueProvider value); @@ -125,7 +125,7 @@ public interface Options order = 4, description = "Output file directory in Cloud Storage", helpText = - "Output directory where output Avro files are archived. Must contain / at the end. For example: `gs://example-bucket/example-directory/`") + "The output directory where output Avro files are archived. Must contain / at the end. For example: gs://example-bucket/example-directory/") @Required ValueProvider getOutputDirectory(); @@ -135,7 +135,7 @@ public interface Options order = 5, optional = true, description = "Output filename prefix of the files to write", - helpText = "Output filename prefix for the Avro files.", + helpText = "The output filename prefix for the Avro files.", regexes = "^[a-zA-Z\\-]+$") @Default.String("output") ValueProvider getOutputFilenamePrefix(); @@ -147,7 +147,7 @@ public interface Options optional = true, description = "Output filename suffix of the files to write", helpText = - "Output filename suffix for the Avro files.") + "The output filename suffix for the Avro files.") @Default.String("") ValueProvider getOutputFilenameSuffix(); @@ -156,7 +156,7 @@ public interface Options @TemplateParameter.GcsWriteFolder( order = 7, description = "Temporary Avro write directory", - helpText = "Directory for temporary Avro files. Must contain / at the end. For example: `gs://example-bucket/example-directory/`.") + helpText = "The directory for temporary Avro files. Must contain / at the end. For example: gs://example-bucket/example-directory/.") @Required ValueProvider getAvroTempDirectory(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java index 87cc1e81a3..4bce28655c 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java @@ -75,7 +75,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 1, optional = true, description = "Whether input CSV files contain a header record.", - helpText = "Boolean to denote whether headers are included in the CSV. Default `false`.") + helpText = "Whether headers are included in the CSV file. Defaults to: false.") @Default.Boolean(false) ValueProvider getContainsHeaders(); @@ -85,7 +85,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 2, description = "Column delimiter of the data files.", helpText = - "The delimiter that the CSV uses.", + "The column delimiter that the CSV file uses.", example = ",") ValueProvider getDelimiter(); @@ -95,7 +95,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 3, description = "CSV Format to use for parsing records.", helpText = - "The CSV format according to Apache Commons CSV format. Default: `Default.`") + "The CSV format according to Apache Commons CSV format. Defaults to: Default.") ValueProvider getCsvFormat(); void setCsvFormat(ValueProvider csvFormat); @@ -106,7 +106,7 @@ public interface CsvPipelineOptions extends PipelineOptions { regexes = {"^(US-ASCII|ISO-8859-1|UTF-8|UTF-16)$"}, description = "CSV file encoding", helpText = - "CSV file encoding.") + "The CSV file character encoding format. Allowed Values are US-ASCII, ISO-8859-1, UTF-8, and UTF-16.") @Default.String("UTF-8") ValueProvider getCsvFileEncoding(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java index 0a1159922c..5014ad112f 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java @@ -85,7 +85,7 @@ public interface SpannerReadOptions extends PipelineOptions { order = 1, regexes = {"^.+$"}, description = "Spanner Table", - helpText = "The table to read the data from.") + helpText = "The Spanner table to read the data from.") ValueProvider getSpannerTable(); @SuppressWarnings("unused") @@ -95,7 +95,7 @@ public interface SpannerReadOptions extends PipelineOptions { order = 2, description = "Read data from Cloud Spanner Project Id", helpText = - "The Google Cloud Project ID of the Spanner database that you want to read data from.") + "The ID of the Google Cloud project that contains the Spanner database to read data from.") ValueProvider getSpannerProjectId(); @SuppressWarnings("unused") @@ -157,7 +157,7 @@ public interface SpannerReadOptions extends PipelineOptions { description = "Use independent compute resource (Spanner DataBoost).", helpText = "Set to `true` to use the compute resources of Spanner Data Boost to run the job with near-zero" - + " impact on Spanner OLTP workflows. This requires the `spanner.databases.useDataBoost` Identity and" + + " impact on Spanner OLTP workflows. When true, requires the `spanner.databases.useDataBoost` Identity and" + " Access Management (IAM) permission. For more information, see" + "Data Boost overview (https://cloud.google.com/spanner/docs/databoost/databoost-overview).") @Default.Boolean(false) ValueProvider getDataBoostEnabled(); diff --git a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java index 45fca623df..ee719628f1 100644 --- a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java +++ b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java @@ -94,7 +94,7 @@ interface WriteOptions extends BigtableCommonOptions { regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Bigtable App Profile", helpText = - "The ID of the Bigtable application profile to be used for the export. If you" + + "The ID of the Bigtable application profile to use for the export. If you" + "do not specify an app profile, Bigtable uses the" + "default app profile (https://cloud.google.com/bigtable/docs/app-profiles#default-app-profile)" + "of the instance.") @Default.String("default") String getBigtableWriteAppProfile(); @@ -106,7 +106,7 @@ interface WriteOptions extends BigtableCommonOptions { optional = true, description = "Bigtable Project ID", helpText = - "The ID of the Google Cloud project of the Bigtable instance that you want" + + "The ID of the Google Cloud project that contains the Bigtable instance" + "to write data to.") String getBigtableWriteProjectId(); @@ -125,7 +125,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 7, optional = true, description = "The max number of row keys in a Bigtable batch write operation", - helpText = "The max number of row keys in a Bigtable batch write operation.") + helpText = "The maximum number of row keys in a Bigtable batch write operation.") Integer getBigtableBulkWriteMaxRowKeyCount(); void setBigtableBulkWriteMaxRowKeyCount(Integer value); @@ -134,7 +134,7 @@ interface WriteOptions extends BigtableCommonOptions { order = 8, optional = true, description = "The max amount of bytes in a Bigtable batch write operation", - helpText = "The max amount of bytes to include per Bigtable batch write operation.") + helpText = "The maximum bytes to include per Bigtable batch write operation.") Integer getBigtableBulkWriteMaxRequestSizeBytes(); void setBigtableBulkWriteMaxRequestSizeBytes(Integer value); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index f238ca09f9..24b6904216 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -180,9 +180,9 @@ public interface BigQueryReadOptions extends PipelineOptions { description = "The dead-letter table name to output failed messages to BigQuery", helpText = "The BigQuery table for messages that failed to reach the output" - + "table, in the format of `my-project:dataset.my-deadletter-table` " - + "If it doesn't exist, the table is created during pipeline execution. If" - + "not specified, `_error_records` is used instead.", + + "table, in the format :..` " + + "If a table doesn't exist, is is created during pipeline execution. If" + + "not specified, `_error_records` is used.", example = "your-project-id:your-dataset.your-table-name") String getOutputDeadletterTable(); @@ -193,7 +193,7 @@ public interface BigQueryReadOptions extends PipelineOptions { optional = true, regexes = {"^.+$"}, description = "Input SQL query.", - helpText = "The SQL query to use to read data from BigQuery. If the BigQuery dataset is in a different project than the Dataflow job, specify the full dataset name in the SQL query, as follows: `projectId.datasetName.tablename`. By default, the `query` parameter uses" + helpText = "The SQL query to use to read data from BigQuery. If the BigQuery dataset is in a different project than the Dataflow job, specify the full dataset name in the SQL query, for example: ... By default, the `query` parameter uses" + "GoogleSQL (https://cloud.google.com/bigquery/docs/introduction-sql)" + ", unless `useLegacySql` is `true`." + "You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "select * from sampledb.sample_table") @@ -205,8 +205,8 @@ public interface BigQueryReadOptions extends PipelineOptions { order = 4, optional = true, description = "Set to true to use legacy SQL", - helpText = "Set to `true` to use legacy SQL. This parameter only applies when using" - + "the `query` parameter. Default: `false`.") + helpText = "Set to true to use legacy SQL. This parameter only applies when using" + + "the `query` parameter. Defaults to: false.") @Default.Boolean(false) Boolean getUseLegacySql(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java index 10b07b34bf..677856df0e 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java @@ -183,7 +183,7 @@ public interface CsvPipelineOptions extends PipelineOptions { @TemplateParameter.GcsReadFile( order = 1, description = "The input filepattern to read from.", - helpText = "The Cloud Storage file pattern to search for CSV files. Example: `gs://mybucket/test-*.csv.`") + helpText = "The Cloud Storage file pattern to search for CSV files. Example: gs://mybucket/test-*.csv.") String getInputFileSpec(); void setInputFileSpec(String inputFileSpec); @@ -237,7 +237,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 6, optional = true, description = "Path to JSON schema", - helpText = "The Path to JSON schema. Default: `null`.", + helpText = "The path to the JSON schema. Defaults to: null.", example = "gs://path/to/schema") String getJsonSchemaPath(); @@ -247,7 +247,7 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 7, optional = true, description = "Set to true if number of files is in the tens of thousands", - helpText = "Set to true if number of files is in the tens of thousands. Default: `false`") + helpText = "Set to true if number of files is in the tens of thousands. Defaults to: false.") @Default.Boolean(false) Boolean getLargeNumFiles(); @@ -259,7 +259,7 @@ public interface CsvPipelineOptions extends PipelineOptions { regexes = {"^(US-ASCII|ISO-8859-1|UTF-8|UTF-16)$"}, description = "CSV file encoding", helpText = - "CSV file encoding.") + "The CSV file character encoding format. Allowed Values are US-ASCII, ISO-8859-1, UTF-8, and UTF-16.") @Default.String("UTF-8") String getCsvFileEncoding(); diff --git a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java index 6373bb639d..6de49155f1 100644 --- a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java +++ b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java @@ -29,7 +29,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 1, description = "Elasticsearch URL or CloudID if using Elastic Cloud", helpText = - "Elasticsearch URL in the format `https://hostname:[port]` or specify CloudID if using Elastic Cloud.", + "The Elasticsearch URL in the format https://hostname:[port]. If using Elastic Cloud, specify the CloudID.", example = "https://elasticsearch-host:9200") @Validation.Required String getConnectionUrl(); @@ -40,7 +40,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 2, description = "Base64 Encoded API Key for access without requiring basic authentication", helpText = - "Base64 Encoded API key used for authentication.") + "The Base64-encoded API key to use for authentication.") @Validation.Required String getApiKey(); @@ -71,7 +71,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = false, regexes = {"[a-zA-Z0-9._-]+"}, description = "Elasticsearch index", - helpText = "The Elasticsearch index toward which the requests will be issued such as `my-index.`", + helpText = "The Elasticsearch index that the requests are issued to, such as `my-index.`", example = "my-index") String getIndex(); @@ -81,7 +81,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 6, optional = true, description = "Batch Size", - helpText = "Batch size in number of documents. Default: `1000`") + helpText = "The batch size in number of documents. Defaults to: 1000.") @Default.Long(1000) Long getBatchSize(); @@ -92,7 +92,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Batch Size in Bytes", helpText = - "Batch size in number of bytes. Default: `5242880` (5mb).") + "The batch size in number of bytes. Defaults to: 5242880 (5mb).") @Default.Long(5242880) Long getBatchSizeBytes(); @@ -102,7 +102,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 8, optional = true, description = "Max retry attempts.", - helpText = "Max retry attempts, must be > 0. Default: no retries.") + helpText = "The maximum number of retry attempts. Must be greater than zero. Defaults to: no retries.") Integer getMaxRetryAttempts(); void setMaxRetryAttempts(Integer maxRetryAttempts); @@ -111,7 +111,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 9, optional = true, description = "Max retry duration.", - helpText = "Max retry duration in milliseconds, must be > 0. Default: no retries.") + helpText = "The maximum retry duration in milliseconds. Must be greater than zero. Defaults to: no retries.") Long getMaxRetryDuration(); void setMaxRetryDuration(Long maxRetryDuration); @@ -121,7 +121,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Document property to specify _index metadata", helpText = - "A property in the document being indexed whose value will specify `_index` metadata to be included with document in bulk request (takes precedence over an `_index` UDF). Default: none.") + "The property in the document being indexed whose value specifies `_index` metadata to include with the document in bulk requests. Takes precedence over an `_index` UDF. Defaults to: none.") String getPropertyAsIndex(); void setPropertyAsIndex(String propertyAsIndex); @@ -131,7 +131,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for _index metadata", helpText = - "The Cloud Storage path to the JavaScript UDF source for a function that will specify `_index` metadata to be included with document in bulk request. Default: none.") + "The Cloud Storage path to the JavaScript UDF source for a function that specifies `_index` metadata to include with the document in bulk requests. Defaults to: none.") String getJavaScriptIndexFnGcsPath(); void setJavaScriptIndexFnGcsPath(String javaScriptTextTransformGcsPath); @@ -141,7 +141,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for _index metadata", helpText = - "UDF JavaScript function name for function that will specify `_index` metadata to be included with document in bulk request. Default: none.") + "The name of the UDF JavaScript function that specifies `_index` metadata to include with the document in bulk requests. Defaults to: none.") String getJavaScriptIndexFnName(); void setJavaScriptIndexFnName(String javaScriptTextTransformFunctionName); @@ -151,7 +151,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Document property to specify _id metadata", helpText = - "A property in the document being indexed whose value will specify `_id` metadata to be included with document in bulk request (takes precedence over an `_id` UDF). Default: none.") + "A property in the document being indexed whose value specifies `_id` metadata to include with the document in bulk requests. Takes precedence over an `_id` UDF. Defaults to: none.") String getPropertyAsId(); void setPropertyAsId(String propertyAsId); @@ -161,7 +161,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for _id metadata", helpText = - "The Cloud Storage path to the JavaScript UDF source for a function that will specify `_id` metadata to be included with document in bulk request. Default: none.") + "The Cloud Storage path to the JavaScript UDF source for the function that specifies `_id` metadata be include with the document in bulk requests. Defaults to: none.") String getJavaScriptIdFnGcsPath(); void setJavaScriptIdFnGcsPath(String javaScriptTextTransformGcsPath); @@ -171,7 +171,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for _id metadata", helpText = - "UDF JavaScript function name for function that will specify `_id` metadata to be included with document in bulk request. Default: none.") + "The name of the UDF JavaScript function that specifies the `_id` metadata to include with the document in bulk requests. Defaults to: none.") String getJavaScriptIdFnName(); void setJavaScriptIdFnName(String javaScriptTextTransformFunctionName); @@ -191,7 +191,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for _type metadata", helpText = - "UDF JavaScript function name for function that will specify `_type` metadata to be included with document in bulk request. Default: none.") + "The name of the UDF JavaScript function that specifies the `_type` metadata to include with the document in bulk requests. Defaults to: none.") String getJavaScriptTypeFnName(); void setJavaScriptTypeFnName(String javaScriptTextTransformFunctionName); @@ -201,7 +201,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for isDelete function", helpText = - "The Cloud Storage path to JavaScript UDF source for function that will determine if document should be deleted rather than inserted or updated. The function should return string value `true` or `false`. Default: none.") + "The Cloud Storage path to the JavaScript UDF source for the function that determines whether to delete the document instead of inserting or updating it. The function returns a string value of \`true\` or \`false\`. Defaults to: none.") String getJavaScriptIsDeleteFnGcsPath(); void setJavaScriptIsDeleteFnGcsPath(String javaScriptTextTransformGcsPath); @@ -211,7 +211,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for isDelete", helpText = - "UDF JavaScript function name for function that will determine if document should be deleted rather than inserted or updated. The function should return string value `true` or `false`. Default: none.") + "The name of the UDF JavaScript function that determines whether to delete the document instead of inserting or updating it. The function returns a string value of \`true\` or \`false\`. Defaults to: none.") String getJavaScriptIsDeleteFnName(); void setJavaScriptIsDeleteFnName(String javaScriptTextTransformFunctionName); @@ -221,7 +221,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Use partial updates", helpText = - "Whether to use partial updates (update rather than create or index, allowing partial docs) with Elasticsearch requests. Default: `false`.") + "Whether to use partial updates (update rather than create or index, allowing partial documents) with Elasticsearch requests. Defaults to: false.") @Default.Boolean(false) Boolean getUsePartialUpdate(); @@ -233,7 +233,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Build insert method", helpText = - "Whether to use `INDEX` (index, allows upserts) or `CREATE` (create, errors on duplicate _id) with Elasticsearch bulk requests. Default: `CREATE`.") + "Whether to use `INDEX` (index, allows upserts) or `CREATE` (create, errors on duplicate _id) with Elasticsearch bulk requests. Defaults to: CREATE.") @Default.Enum("CREATE") BulkInsertMethodOptions getBulkInsertMethod(); diff --git a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java index 01245f8513..540581cdf3 100644 --- a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java +++ b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java @@ -34,7 +34,7 @@ public interface GCSToElasticsearchOptions order = 1, description = "BigQuery Deadletter table to send failed inserts.", helpText = - "The BigQuery Deadletter table to send failed inserts to.", + "The BigQuery dead-letter table to send failed inserts to.", example = "your-project:your-dataset.your-table-name") @Validation.Required String getDeadletterTable(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java index fd6ed6cb3e..6a4c4ef1e0 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java @@ -33,7 +33,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Spanner Project ID", helpText = - "The project to read change streams from. This is also the project where the change " + "The project to read change streams from. This project is also where the change " + "streams connector metadata table is created. The default for this parameter is the project where the Dataflow pipeline is running.") @Default.String("") String getSpannerProjectId(); @@ -84,7 +84,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt order = 6, description = "Spanner metadata database", helpText = - "TThe Spanner database to use for the change streams connector metadata table.") + "The Spanner database to use for the change streams connector metadata table.") @Validation.Required String getSpannerMetadataDatabase(); @@ -164,7 +164,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Pub/Sub API", helpText = - "Pub/Sub API used to implement the pipeline. Allowed APIs are `pubsubio` and `native_client`." + "The Pub/Sub API used to implement the pipeline. Allowed APIs are `pubsubio` and `native_client`." + " For a small number of queries per second (QPS), `native_client` has less latency. For a" + " large number of QPS, `pubsubio` provides better and more stable performance. The default is `pubsubio`.") @Default.String("pubsubio") @@ -203,7 +203,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt optional = true, description = "Priority for Spanner RPC invocations", helpText = - "The request priority for Spanner calls. The value must be one of: [HIGH,MEDIUM,LOW]. (Default: HIGH)") + "The request priority for Spanner calls. Allowed values are HIGH, MEDIUM, and LOW. Defaults to: HIGH)") @Default.Enum("HIGH") RpcPriority getRpcPriority(); diff --git a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java index 029f67e710..cca5c761b2 100644 --- a/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java +++ b/v2/jdbc-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/JdbcToPubsubOptions.java @@ -54,7 +54,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { regexes = {"^.+$"}, description = "JDBC connection username.", helpText = - "The username to be used for the JDBC connection. You can pass in this value encrypted by a Cloud KMS key as a Base64-encoded string. For example, `echo -n 'some_username' | glcloud kms encrypt --location=my_location --keyring=mykeyring --key=mykey --plaintext-file=- --ciphertext-file=- | base64`") + "The username to use for the JDBC connection. You can pass in this value as a string that's encrypted with a Cloud KMS key and then Base64-encoded. For example, `echo -n 'some_username' | glcloud kms encrypt --location=my_location --keyring=mykeyring --key=mykey --plaintext-file=- --ciphertext-file=- | base64`") String getUsername(); void setUsername(String username); @@ -64,7 +64,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = true, description = "JDBC connection password.", helpText = - "The password to be used for the JDBC connection. You can pass in this value encrypted by a Cloud KMS key as a Base64-encoded string. For example, `echo -n 'some_password' | glcloud kms encrypt --location=my_location --keyring=mykeyring --key=mykey --plaintext-file=- --ciphertext-file=- | base64`") + "The password to use for the JDBC connection. You can pass in this value as a string that's encrypted with a Cloud KMS key and then Base64-encoded. For example, `echo -n 'some_password' | glcloud kms encrypt --location=my_location --keyring=mykeyring --key=mykey --plaintext-file=- --ciphertext-file=- | base64`") String getPassword(); void setPassword(String password); @@ -74,7 +74,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = false, regexes = {"^.+$"}, description = "Cloud Storage paths for JDBC drivers", - helpText = "Comma separated Cloud Storage paths for JDBC drivers.", + helpText = "Comma-separated Cloud Storage paths for JDBC drivers.", example = "gs://your-bucket/driver_jar1.jar,gs://your-bucket/driver_jar2.jar") String getDriverJars(); @@ -86,7 +86,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { regexes = {"^[a-zA-Z0-9_;!*&=@#-:\\/]+$"}, description = "JDBC connection property string.", helpText = - "Properties string to use for the JDBC connection. Format of the string must be `[propertyName=property;]*`. ", + "The properties string to use for the JDBC connection. The format of the string must be `[propertyName=property;]*`. ", example = "unicode=true;characterEncoding=UTF-8") String getConnectionProperties(); @@ -97,7 +97,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = false, regexes = {"^.+$"}, description = "JDBC source SQL query.", - helpText = "The query to be run on the source to extract the data.", + helpText = "The query to run on the source to extract the data.", example = "select * from sampledb.sample_table") String getQuery(); @@ -107,7 +107,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { order = 8, description = "Output Pub/Sub topic", helpText = - "The Pub/Sub topic to publish to, in the format of `projects//topics/`.", + "The Pub/Sub topic to publish to, in the format projects//topics/.", example = "projects/your-project-id/topics/your-topic-name") @Validation.Required String getOutputTopic(); @@ -119,7 +119,7 @@ public interface JdbcToPubsubOptions extends CommonTemplateOptions { optional = true, description = "Google Cloud KMS key", helpText = - "Cloud KMS Encryption Key to decrypt the username, password, and connection string. If Cloud KMS key is passed in, the username, password, and connection string must all be passed in encrypted and base64 encoded.", + "The Cloud KMS Encryption Key to use to decrypt the username, password, and connection string. If a Cloud KMS key is passed in, the username, password, and connection string must all be passed in encrypted and base64 encoded.", example = "projects/your-project/locations/global/keyRings/your-keyring/cryptoKeys/your-key") String getKMSEncryptionKey(); diff --git a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java index 0d6d7d7800..7d796e7dfd 100644 --- a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java +++ b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java @@ -174,7 +174,7 @@ public interface KafkaToBQOptions optional = true, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", - helpText = "The host address of the running Apache Kafka broker servers in a comma-separated list, each host address in the format of `35.70.252.199:9092`", + helpText = "The host address of the running Apache Kafka broker servers in a comma-separated list. Each host address must be in the format `35.70.252.199:9092`.", example = "localhost:9092,127.0.0.1:9093") @Deprecated String getBootstrapServers(); diff --git a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java index 58ddbdf40e..69d5f853ae 100644 --- a/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java +++ b/v2/mongodb-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/mongodb/options/MongoDbToBigQueryOptions.java @@ -86,7 +86,7 @@ public interface PubSubOptions extends PipelineOptions, DataflowPipelineOptions order = 1, description = "Pub/Sub input topic", helpText = - "The Pub/Sub input topic to read from, in the format of `projects//topics/`.") + "The Pub/Sub input topic to read from, in the format of projects//topics/.") String getInputTopic(); void setInputTopic(String inputTopic); diff --git a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java index c8b3fcabe3..0e232b25e7 100644 --- a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java +++ b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java @@ -136,7 +136,7 @@ public interface PubSubToRedisOptions description = "Pub/Sub input subscription", helpText = "The Pub/Sub subscription to read the input from, in the format of" - + " `'projects/my-project-id/subscriptions/my-subscription-id'`.", + + " projects//subscriptions/.", example = "projects/your-project-id/subscriptions/your-subscription-name") String getInputSubscription(); From d539607b70e57ce22009b1d7d77c6cf65e371384 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Wed, 15 May 2024 10:15:23 +0530 Subject: [PATCH 04/16] Added changes based on comments --- .../cloud/teleport/templates/CSVToBigQuery.java | 5 +++-- .../cloud/teleport/templates/SpannerToText.java | 3 ++- .../templates/common/SpannerConverters.java | 8 +++++--- .../bigtable/options/BigtableCommonOptions.java | 6 ++++-- .../v2/transforms/BigQueryConverters.java | 15 +++++++++------ 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java b/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java index 84d9dd32ea..6438c1a401 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java @@ -143,8 +143,9 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { description = "BigQuery output table for bad records", helpText = "The name of the BigQuery table to use to store the rejected data when processing the" - + " CSV files. If you reuse an existing BigQuery table, the data is appended to the" - + " destination table. The schema of this table must match the" + "error table schema (https://cloud.google.com/dataflow/docs/guides/templates/provided/cloud-storage-csv-to-bigquery#GcsCSVToBigQueryBadRecordsSchema).") + + " CSV files. If you reuse an existing BigQuery table, the data is appended to the" + + " destination table. The schema of this table must match the" + + " error table schema (https://cloud.google.com/dataflow/docs/guides/templates/provided/cloud-storage-csv-to-bigquery#GcsCSVToBigQueryBadRecordsSchema).") ValueProvider getBadRecordsOutputTable(); void setBadRecordsOutputTable(ValueProvider value); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java b/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java index 193490edb8..5c07368ddd 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java @@ -105,7 +105,8 @@ public interface SpannerToTextOptions optional = true, description = "Priority for Spanner RPC invocations", helpText = - "The" + "request priority (https://cloud.google.com/spanner/docs/reference/rest/v1/RequestOptions)" + "for Spanner calls. Possible values are `HIGH`, `MEDIUM`, `LOW`. The default value is `MEDIUM`.") + "The request priority (https://cloud.google.com/spanner/docs/reference/rest/v1/RequestOptions)" + + " for Spanner calls. Possible values are `HIGH`, `MEDIUM`, `LOW`. The default value is `MEDIUM`.") ValueProvider getSpannerPriority(); void setSpannerPriority(ValueProvider value); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java index 5014ad112f..bb4855a20e 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java @@ -142,8 +142,9 @@ public interface SpannerReadOptions extends PipelineOptions { description = "Snapshot time", helpText = "The timestamp that corresponds to the version of the Spanner database that you want to read." - + " The timestamp must be specified as per" + "RFC 3339 (https://tools.ietf.org/html/rfc3339)" + "UTC \"Zulu\" format. " - + " The timestamp must be in the past and" + "Maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness)" + "applies.", + + " The timestamp must be specified as per RFC 3339 (https://tools.ietf.org/html/rfc3339) UTC \"Zulu\" format." + + " The timestamp must be in the past and" + + " Maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness) applies.", example = "1990-12-31T23:59:60Z") @Default.String(value = "") ValueProvider getSpannerSnapshotTime(); @@ -158,7 +159,8 @@ public interface SpannerReadOptions extends PipelineOptions { helpText = "Set to `true` to use the compute resources of Spanner Data Boost to run the job with near-zero" + " impact on Spanner OLTP workflows. When true, requires the `spanner.databases.useDataBoost` Identity and" - + " Access Management (IAM) permission. For more information, see" + "Data Boost overview (https://cloud.google.com/spanner/docs/databoost/databoost-overview).") + + " Access Management (IAM) permission. For more information, see" + + " Data Boost overview (https://cloud.google.com/spanner/docs/databoost/databoost-overview).") @Default.Boolean(false) ValueProvider getDataBoostEnabled(); diff --git a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java index ee719628f1..eb9f9618d2 100644 --- a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java +++ b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java @@ -94,8 +94,10 @@ interface WriteOptions extends BigtableCommonOptions { regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Bigtable App Profile", helpText = - "The ID of the Bigtable application profile to use for the export. If you" + - "do not specify an app profile, Bigtable uses the" + "default app profile (https://cloud.google.com/bigtable/docs/app-profiles#default-app-profile)" + "of the instance.") + "The ID of the Bigtable application profile to use for the export. If you" + + " do not specify an app profile, Bigtable uses the" + + " default app profile (https://cloud.google.com/bigtable/docs/app-profiles#default-app-profile)" + + " of the instance.") @Default.String("default") String getBigtableWriteAppProfile(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index 24b6904216..c831f3a0ee 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -166,9 +166,11 @@ public interface BigQueryReadOptions extends PipelineOptions { order = 1, optional = true, description = "BigQuery source table", - helpText = "The BigQuery table to read from. Format: `projectId:datasetId.tablename`. If you specify `inputTableSpec`, the template reads the data directly from BigQuery storage by using the" + "BigQuery Storage Read API. (https://cloud.google.com/bigquery/docs/reference/storage)" + "For information about limitations in the Storage Read API, see" - + "Limitations.(https://cloud.google.com/bigquery/docs/reference/storage#limitations)" - + "You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", + helpText = "The BigQuery table to read from. Format: `projectId:datasetId.tablename`. If you specify `inputTableSpec`, the template reads the data directly from BigQuery storage by using the" + + " BigQuery Storage Read API. (https://cloud.google.com/bigquery/docs/reference/storage)" + + " For information about limitations in the Storage Read API, see" + + " Limitations.(https://cloud.google.com/bigquery/docs/reference/storage#limitations)" + + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "bigquery-project:dataset.input_table") String getInputTableSpec(); @@ -193,9 +195,10 @@ public interface BigQueryReadOptions extends PipelineOptions { optional = true, regexes = {"^.+$"}, description = "Input SQL query.", - helpText = "The SQL query to use to read data from BigQuery. If the BigQuery dataset is in a different project than the Dataflow job, specify the full dataset name in the SQL query, for example: ... By default, the `query` parameter uses" - + "GoogleSQL (https://cloud.google.com/bigquery/docs/introduction-sql)" + ", unless `useLegacySql` is `true`." - + "You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", + helpText = "The SQL query to use to read data from BigQuery. If the BigQuery dataset is in a different project than the Dataflow job," + + " specify the full dataset name in the SQL query, for example: ... By default, the `query` parameter uses" + + " GoogleSQL (https://cloud.google.com/bigquery/docs/introduction-sql), unless `useLegacySql` is `true`." + + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "select * from sampledb.sample_table") String getQuery(); From cb898863a60cb9e486d580fbc58196ad3cd34589 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Wed, 15 May 2024 10:21:29 +0530 Subject: [PATCH 05/16] spotless apply --- .../teleport/bigtable/BigtableToJson.java | 3 ++- .../teleport/templates/CSVToBigQuery.java | 9 +++---- .../teleport/templates/PubsubToAvro.java | 6 ++--- .../teleport/templates/SpannerToText.java | 2 +- .../templates/common/CsvConverters.java | 6 ++--- .../templates/common/SpannerConverters.java | 4 +-- .../options/BigtableCommonOptions.java | 12 ++++----- .../v2/transforms/BigQueryConverters.java | 25 +++++++++++-------- .../teleport/v2/transforms/CsvConverters.java | 6 +++-- .../options/ElasticsearchWriteOptions.java | 16 ++++++------ .../options/GCSToElasticsearchOptions.java | 3 +-- .../SpannerChangeStreamsToPubSubOptions.java | 3 +-- .../v2/templates/KafkaToBigQuery.java | 3 ++- 13 files changed, 50 insertions(+), 48 deletions(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java b/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java index df3ab89241..381b948cc1 100644 --- a/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java +++ b/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java @@ -116,7 +116,8 @@ public interface Options extends PipelineOptions { @TemplateParameter.Text( order = 5, description = "JSON file prefix", - helpText = "The prefix of the JSON file name. For example, \"table1-\". If no value is provided, defaults to `part`.") + helpText = + "The prefix of the JSON file name. For example, \"table1-\". If no value is provided, defaults to `part`.") @Default.String("part") ValueProvider getFilenamePrefix(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java b/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java index 6438c1a401..cd3d809acc 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/CSVToBigQuery.java @@ -112,8 +112,7 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { @TemplateParameter.GcsReadFile( order = 2, description = "Cloud Storage location of your BigQuery schema file, described as a JSON", - helpText = - "The Cloud Storage path to the JSON file that defines your BigQuery schema.") + helpText = "The Cloud Storage path to the JSON file that defines your BigQuery schema.") ValueProvider getSchemaJSONPath(); void setSchemaJSONPath(ValueProvider value); @@ -143,9 +142,9 @@ public interface Options extends DataflowPipelineOptions, CsvPipelineOptions { description = "BigQuery output table for bad records", helpText = "The name of the BigQuery table to use to store the rejected data when processing the" - + " CSV files. If you reuse an existing BigQuery table, the data is appended to the" - + " destination table. The schema of this table must match the" - + " error table schema (https://cloud.google.com/dataflow/docs/guides/templates/provided/cloud-storage-csv-to-bigquery#GcsCSVToBigQueryBadRecordsSchema).") + + " CSV files. If you reuse an existing BigQuery table, the data is appended to the" + + " destination table. The schema of this table must match the" + + " error table schema (https://cloud.google.com/dataflow/docs/guides/templates/provided/cloud-storage-csv-to-bigquery#GcsCSVToBigQueryBadRecordsSchema).") ValueProvider getBadRecordsOutputTable(); void setBadRecordsOutputTable(ValueProvider value); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java b/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java index 54195301ea..78f94d1dce 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/PubsubToAvro.java @@ -146,8 +146,7 @@ public interface Options order = 6, optional = true, description = "Output filename suffix of the files to write", - helpText = - "The output filename suffix for the Avro files.") + helpText = "The output filename suffix for the Avro files.") @Default.String("") ValueProvider getOutputFilenameSuffix(); @@ -156,7 +155,8 @@ public interface Options @TemplateParameter.GcsWriteFolder( order = 7, description = "Temporary Avro write directory", - helpText = "The directory for temporary Avro files. Must contain / at the end. For example: gs://example-bucket/example-directory/.") + helpText = + "The directory for temporary Avro files. Must contain / at the end. For example: gs://example-bucket/example-directory/.") @Required ValueProvider getAvroTempDirectory(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java b/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java index 5c07368ddd..98c7642651 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java @@ -105,7 +105,7 @@ public interface SpannerToTextOptions optional = true, description = "Priority for Spanner RPC invocations", helpText = - "The request priority (https://cloud.google.com/spanner/docs/reference/rest/v1/RequestOptions)" + "The request priority (https://cloud.google.com/spanner/docs/reference/rest/v1/RequestOptions)" + " for Spanner calls. Possible values are `HIGH`, `MEDIUM`, `LOW`. The default value is `MEDIUM`.") ValueProvider getSpannerPriority(); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java index 4bce28655c..04739b3f60 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/CsvConverters.java @@ -84,8 +84,7 @@ public interface CsvPipelineOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, description = "Column delimiter of the data files.", - helpText = - "The column delimiter that the CSV file uses.", + helpText = "The column delimiter that the CSV file uses.", example = ",") ValueProvider getDelimiter(); @@ -94,8 +93,7 @@ public interface CsvPipelineOptions extends PipelineOptions { @TemplateParameter.Text( order = 3, description = "CSV Format to use for parsing records.", - helpText = - "The CSV format according to Apache Commons CSV format. Defaults to: Default.") + helpText = "The CSV format according to Apache Commons CSV format. Defaults to: Default.") ValueProvider getCsvFormat(); void setCsvFormat(ValueProvider csvFormat); diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java index bb4855a20e..dd7b15b47b 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java @@ -143,7 +143,7 @@ public interface SpannerReadOptions extends PipelineOptions { helpText = "The timestamp that corresponds to the version of the Spanner database that you want to read." + " The timestamp must be specified as per RFC 3339 (https://tools.ietf.org/html/rfc3339) UTC \"Zulu\" format." - + " The timestamp must be in the past and" + + " The timestamp must be in the past and" + " Maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness) applies.", example = "1990-12-31T23:59:60Z") @Default.String(value = "") @@ -159,7 +159,7 @@ public interface SpannerReadOptions extends PipelineOptions { helpText = "Set to `true` to use the compute resources of Spanner Data Boost to run the job with near-zero" + " impact on Spanner OLTP workflows. When true, requires the `spanner.databases.useDataBoost` Identity and" - + " Access Management (IAM) permission. For more information, see" + + " Access Management (IAM) permission. For more information, see" + " Data Boost overview (https://cloud.google.com/spanner/docs/databoost/databoost-overview).") @Default.Boolean(false) ValueProvider getDataBoostEnabled(); diff --git a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java index eb9f9618d2..949543c75a 100644 --- a/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java +++ b/v2/bigtable-common/src/main/java/com/google/cloud/teleport/v2/bigtable/options/BigtableCommonOptions.java @@ -94,10 +94,10 @@ interface WriteOptions extends BigtableCommonOptions { regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Bigtable App Profile", helpText = - "The ID of the Bigtable application profile to use for the export. If you" - + " do not specify an app profile, Bigtable uses the" - + " default app profile (https://cloud.google.com/bigtable/docs/app-profiles#default-app-profile)" - + " of the instance.") + "The ID of the Bigtable application profile to use for the export. If you" + + " do not specify an app profile, Bigtable uses the" + + " default app profile (https://cloud.google.com/bigtable/docs/app-profiles#default-app-profile)" + + " of the instance.") @Default.String("default") String getBigtableWriteAppProfile(); @@ -108,8 +108,8 @@ interface WriteOptions extends BigtableCommonOptions { optional = true, description = "Bigtable Project ID", helpText = - "The ID of the Google Cloud project that contains the Bigtable instance" + - "to write data to.") + "The ID of the Google Cloud project that contains the Bigtable instance" + + "to write data to.") String getBigtableWriteProjectId(); void setBigtableWriteProjectId(String value); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index c831f3a0ee..6a57f3bfe5 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -166,11 +166,12 @@ public interface BigQueryReadOptions extends PipelineOptions { order = 1, optional = true, description = "BigQuery source table", - helpText = "The BigQuery table to read from. Format: `projectId:datasetId.tablename`. If you specify `inputTableSpec`, the template reads the data directly from BigQuery storage by using the" - + " BigQuery Storage Read API. (https://cloud.google.com/bigquery/docs/reference/storage)" - + " For information about limitations in the Storage Read API, see" - + " Limitations.(https://cloud.google.com/bigquery/docs/reference/storage#limitations)" - + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", + helpText = + "The BigQuery table to read from. Format: `projectId:datasetId.tablename`. If you specify `inputTableSpec`, the template reads the data directly from BigQuery storage by using the" + + " BigQuery Storage Read API. (https://cloud.google.com/bigquery/docs/reference/storage)" + + " For information about limitations in the Storage Read API, see" + + " Limitations.(https://cloud.google.com/bigquery/docs/reference/storage#limitations)" + + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "bigquery-project:dataset.input_table") String getInputTableSpec(); @@ -195,10 +196,11 @@ public interface BigQueryReadOptions extends PipelineOptions { optional = true, regexes = {"^.+$"}, description = "Input SQL query.", - helpText = "The SQL query to use to read data from BigQuery. If the BigQuery dataset is in a different project than the Dataflow job," - + " specify the full dataset name in the SQL query, for example: ... By default, the `query` parameter uses" - + " GoogleSQL (https://cloud.google.com/bigquery/docs/introduction-sql), unless `useLegacySql` is `true`." - + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", + helpText = + "The SQL query to use to read data from BigQuery. If the BigQuery dataset is in a different project than the Dataflow job," + + " specify the full dataset name in the SQL query, for example: ... By default, the `query` parameter uses" + + " GoogleSQL (https://cloud.google.com/bigquery/docs/introduction-sql), unless `useLegacySql` is `true`." + + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "select * from sampledb.sample_table") String getQuery(); @@ -208,8 +210,9 @@ public interface BigQueryReadOptions extends PipelineOptions { order = 4, optional = true, description = "Set to true to use legacy SQL", - helpText = "Set to true to use legacy SQL. This parameter only applies when using" - + "the `query` parameter. Defaults to: false.") + helpText = + "Set to true to use legacy SQL. This parameter only applies when using" + + "the `query` parameter. Defaults to: false.") @Default.Boolean(false) Boolean getUseLegacySql(); diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java index 677856df0e..7440101d41 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/CsvConverters.java @@ -183,7 +183,8 @@ public interface CsvPipelineOptions extends PipelineOptions { @TemplateParameter.GcsReadFile( order = 1, description = "The input filepattern to read from.", - helpText = "The Cloud Storage file pattern to search for CSV files. Example: gs://mybucket/test-*.csv.") + helpText = + "The Cloud Storage file pattern to search for CSV files. Example: gs://mybucket/test-*.csv.") String getInputFileSpec(); void setInputFileSpec(String inputFileSpec); @@ -247,7 +248,8 @@ public interface CsvPipelineOptions extends PipelineOptions { order = 7, optional = true, description = "Set to true if number of files is in the tens of thousands", - helpText = "Set to true if number of files is in the tens of thousands. Defaults to: false.") + helpText = + "Set to true if number of files is in the tens of thousands. Defaults to: false.") @Default.Boolean(false) Boolean getLargeNumFiles(); diff --git a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java index 6de49155f1..ab11468ace 100644 --- a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java +++ b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java @@ -39,8 +39,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { @TemplateParameter.Text( order = 2, description = "Base64 Encoded API Key for access without requiring basic authentication", - helpText = - "The Base64-encoded API key to use for authentication.") + helpText = "The Base64-encoded API key to use for authentication.") @Validation.Required String getApiKey(); @@ -91,8 +90,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 7, optional = true, description = "Batch Size in Bytes", - helpText = - "The batch size in number of bytes. Defaults to: 5242880 (5mb).") + helpText = "The batch size in number of bytes. Defaults to: 5242880 (5mb).") @Default.Long(5242880) Long getBatchSizeBytes(); @@ -102,7 +100,8 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 8, optional = true, description = "Max retry attempts.", - helpText = "The maximum number of retry attempts. Must be greater than zero. Defaults to: no retries.") + helpText = + "The maximum number of retry attempts. Must be greater than zero. Defaults to: no retries.") Integer getMaxRetryAttempts(); void setMaxRetryAttempts(Integer maxRetryAttempts); @@ -111,7 +110,8 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { order = 9, optional = true, description = "Max retry duration.", - helpText = "The maximum retry duration in milliseconds. Must be greater than zero. Defaults to: no retries.") + helpText = + "The maximum retry duration in milliseconds. Must be greater than zero. Defaults to: no retries.") Long getMaxRetryDuration(); void setMaxRetryDuration(Long maxRetryDuration); @@ -201,7 +201,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for isDelete function", helpText = - "The Cloud Storage path to the JavaScript UDF source for the function that determines whether to delete the document instead of inserting or updating it. The function returns a string value of \`true\` or \`false\`. Defaults to: none.") + "The Cloud Storage path to the JavaScript UDF source for the function that determines whether to delete the document instead of inserting or updating it. The function returns a string value of `true` or `false`. Defaults to: none.") String getJavaScriptIsDeleteFnGcsPath(); void setJavaScriptIsDeleteFnGcsPath(String javaScriptTextTransformGcsPath); @@ -211,7 +211,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "UDF JavaScript Function Name for isDelete", helpText = - "The name of the UDF JavaScript function that determines whether to delete the document instead of inserting or updating it. The function returns a string value of \`true\` or \`false\`. Defaults to: none.") + "The name of the UDF JavaScript function that determines whether to delete the document instead of inserting or updating it. The function returns a string value of `true` or `false`. Defaults to: none.") String getJavaScriptIsDeleteFnName(); void setJavaScriptIsDeleteFnName(String javaScriptTextTransformFunctionName); diff --git a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java index 540581cdf3..b5d1e728db 100644 --- a/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java +++ b/v2/googlecloud-to-elasticsearch/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/GCSToElasticsearchOptions.java @@ -33,8 +33,7 @@ public interface GCSToElasticsearchOptions @TemplateParameter.BigQueryTable( order = 1, description = "BigQuery Deadletter table to send failed inserts.", - helpText = - "The BigQuery dead-letter table to send failed inserts to.", + helpText = "The BigQuery dead-letter table to send failed inserts to.", example = "your-project:your-dataset.your-table-name") @Validation.Required String getDeadletterTable(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java index 6a4c4ef1e0..60cf4d4750 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/options/SpannerChangeStreamsToPubSubOptions.java @@ -83,8 +83,7 @@ public interface SpannerChangeStreamsToPubSubOptions extends DataflowPipelineOpt @TemplateParameter.Text( order = 6, description = "Spanner metadata database", - helpText = - "The Spanner database to use for the change streams connector metadata table.") + helpText = "The Spanner database to use for the change streams connector metadata table.") @Validation.Required String getSpannerMetadataDatabase(); diff --git a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java index 7d796e7dfd..0eb08e59e4 100644 --- a/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java +++ b/v2/kafka-to-bigquery/src/main/java/com/google/cloud/teleport/v2/templates/KafkaToBigQuery.java @@ -174,7 +174,8 @@ public interface KafkaToBQOptions optional = true, regexes = {"[,:a-zA-Z0-9._-]+"}, description = "Kafka Bootstrap Server list", - helpText = "The host address of the running Apache Kafka broker servers in a comma-separated list. Each host address must be in the format `35.70.252.199:9092`.", + helpText = + "The host address of the running Apache Kafka broker servers in a comma-separated list. Each host address must be in the format `35.70.252.199:9092`.", example = "localhost:9092,127.0.0.1:9093") @Deprecated String getBootstrapServers(); From 8ffccb381033543ecdbdfe4b32307353c9088a55 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Wed, 15 May 2024 15:20:34 +0530 Subject: [PATCH 06/16] fixing spaces --- .../cloud/teleport/v2/transforms/BigQueryConverters.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index 6a57f3bfe5..c1d910aa62 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -183,9 +183,9 @@ public interface BigQueryReadOptions extends PipelineOptions { description = "The dead-letter table name to output failed messages to BigQuery", helpText = "The BigQuery table for messages that failed to reach the output" - + "table, in the format :..` " - + "If a table doesn't exist, is is created during pipeline execution. If" - + "not specified, `_error_records` is used.", + + " table, in the format :..`" + + " If a table doesn't exist, is is created during pipeline execution. If" + + " not specified, `_error_records` is used.", example = "your-project-id:your-dataset.your-table-name") String getOutputDeadletterTable(); @@ -212,7 +212,7 @@ public interface BigQueryReadOptions extends PipelineOptions { description = "Set to true to use legacy SQL", helpText = "Set to true to use legacy SQL. This parameter only applies when using" - + "the `query` parameter. Defaults to: false.") + + " the `query` parameter. Defaults to: false.") @Default.Boolean(false) Boolean getUseLegacySql(); From aa3bf9fa78f4ddf5e1343e92629e68849536ed85 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:02:48 +0530 Subject: [PATCH 07/16] Update v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../java/com/google/cloud/teleport/bigtable/BigtableToJson.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java b/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java index 381b948cc1..7c787ce4ff 100644 --- a/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java +++ b/v1/src/main/java/com/google/cloud/teleport/bigtable/BigtableToJson.java @@ -86,7 +86,7 @@ public interface Options extends PipelineOptions { order = 2, regexes = {"[a-z][a-z0-9\\-]+[a-z0-9]"}, description = "Instance ID", - helpText = "The ID of the Bigtable instance that contains the table") + helpText = "The ID of the Bigtable instance that contains the table.") ValueProvider getBigtableInstanceId(); @SuppressWarnings("unused") From 279b28ebe79638e5968d28b17a5133944acaa5a1 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:02:58 +0530 Subject: [PATCH 08/16] Update v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../cloud/teleport/templates/common/SpannerConverters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java index dd7b15b47b..eab71ff6bc 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java @@ -141,7 +141,7 @@ public interface SpannerReadOptions extends PipelineOptions { }, description = "Snapshot time", helpText = - "The timestamp that corresponds to the version of the Spanner database that you want to read." + "The timestamp that corresponds to the version of the Spanner database that you want to read from." + " The timestamp must be specified as per RFC 3339 (https://tools.ietf.org/html/rfc3339) UTC \"Zulu\" format." + " The timestamp must be in the past and" + " Maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness) applies.", From c33f5b73e24eb60c58d8d2eccae0f9b69cab7879 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:03:10 +0530 Subject: [PATCH 09/16] Update v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../google/cloud/teleport/v2/transforms/BigQueryConverters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index c1d910aa62..5713e51692 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -168,7 +168,7 @@ public interface BigQueryReadOptions extends PipelineOptions { description = "BigQuery source table", helpText = "The BigQuery table to read from. Format: `projectId:datasetId.tablename`. If you specify `inputTableSpec`, the template reads the data directly from BigQuery storage by using the" - + " BigQuery Storage Read API. (https://cloud.google.com/bigquery/docs/reference/storage)" + + " BigQuery Storage Read API (https://cloud.google.com/bigquery/docs/reference/storage)." + " For information about limitations in the Storage Read API, see" + " Limitations.(https://cloud.google.com/bigquery/docs/reference/storage#limitations)" + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", From de33b1bcb3c068d40f1ba10f91a9bebb55cf56f5 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:03:17 +0530 Subject: [PATCH 10/16] Update v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../google/cloud/teleport/v2/transforms/BigQueryConverters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index 5713e51692..0dd3d6e670 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -170,7 +170,7 @@ public interface BigQueryReadOptions extends PipelineOptions { "The BigQuery table to read from. Format: `projectId:datasetId.tablename`. If you specify `inputTableSpec`, the template reads the data directly from BigQuery storage by using the" + " BigQuery Storage Read API (https://cloud.google.com/bigquery/docs/reference/storage)." + " For information about limitations in the Storage Read API, see" - + " Limitations.(https://cloud.google.com/bigquery/docs/reference/storage#limitations)" + + " https://cloud.google.com/bigquery/docs/reference/storage#limitations." + " You must specify either `inputTableSpec` or `query`. If you set both parameters, the template uses the `query` parameter.", example = "bigquery-project:dataset.input_table") String getInputTableSpec(); From 3daa6bbc5e5345bf111cebb58d47471a46c99352 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:03:34 +0530 Subject: [PATCH 11/16] Update v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../google/cloud/teleport/v2/transforms/BigQueryConverters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java index 0dd3d6e670..53de64259c 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/BigQueryConverters.java @@ -183,7 +183,7 @@ public interface BigQueryReadOptions extends PipelineOptions { description = "The dead-letter table name to output failed messages to BigQuery", helpText = "The BigQuery table for messages that failed to reach the output" - + " table, in the format :..`" + + " table, in the format :.." + " If a table doesn't exist, is is created during pipeline execution. If" + " not specified, `_error_records` is used.", example = "your-project-id:your-dataset.your-table-name") From 5f5561d33b2ae832d1d27e40294d065906fcd641 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:03:40 +0530 Subject: [PATCH 12/16] Update v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../v2/elasticsearch/options/ElasticsearchWriteOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java index ab11468ace..a998fa4fe9 100644 --- a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java +++ b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java @@ -181,7 +181,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for _type metadata", helpText = - "The Cloud Storage path to the JavaScript UDF source for a function that will specify `_type` metadata to be included with document in bulk request. Default: none.") + "The Cloud Storage path to the JavaScript UDF source for a function that specifies `_type` metadata to include with documents in bulk requests. Default: none.") String getJavaScriptTypeFnGcsPath(); void setJavaScriptTypeFnGcsPath(String javaScriptTextTransformGcsPath); From 0efa34440402038832036d2599b0feb5e555597a Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:03:47 +0530 Subject: [PATCH 13/16] Update v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../com/google/cloud/teleport/v2/templates/PubSubToRedis.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java index 0e232b25e7..fdc404834e 100644 --- a/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java +++ b/v2/pubsub-to-redis/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToRedis.java @@ -135,7 +135,7 @@ public interface PubSubToRedisOptions order = 1, description = "Pub/Sub input subscription", helpText = - "The Pub/Sub subscription to read the input from, in the format of" + "The Pub/Sub subscription to read the input from, in the format" + " projects//subscriptions/.", example = "projects/your-project-id/subscriptions/your-subscription-name") String getInputSubscription(); From 4c271be9b9526bc457d7b3d70f640ae23584dfe7 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:04:16 +0530 Subject: [PATCH 14/16] Update v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../cloud/teleport/templates/common/SpannerConverters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java index eab71ff6bc..7f556c18ea 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java @@ -144,7 +144,7 @@ public interface SpannerReadOptions extends PipelineOptions { "The timestamp that corresponds to the version of the Spanner database that you want to read from." + " The timestamp must be specified as per RFC 3339 (https://tools.ietf.org/html/rfc3339) UTC \"Zulu\" format." + " The timestamp must be in the past and" - + " Maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness) applies.", + + " maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness) applies.", example = "1990-12-31T23:59:60Z") @Default.String(value = "") ValueProvider getSpannerSnapshotTime(); From 6f8d8e10ac625e9a0028fd3566a3bef314f259fa Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 20:04:28 +0530 Subject: [PATCH 15/16] Update v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../cloud/teleport/templates/common/SpannerConverters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java index 7f556c18ea..c90eaaee93 100644 --- a/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java +++ b/v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java @@ -142,7 +142,7 @@ public interface SpannerReadOptions extends PipelineOptions { description = "Snapshot time", helpText = "The timestamp that corresponds to the version of the Spanner database that you want to read from." - + " The timestamp must be specified as per RFC 3339 (https://tools.ietf.org/html/rfc3339) UTC \"Zulu\" format." + + " The timestamp must be specified in the RFC 3339 (https://tools.ietf.org/html/rfc3339) UTC \"Zulu\" format." + " The timestamp must be in the past and" + " maximum timestamp staleness (https://cloud.google.com/spanner/docs/timestamp-bounds#maximum_timestamp_staleness) applies.", example = "1990-12-31T23:59:60Z") From 5a53bed049f3ac56719743c6f4d184c747f014a8 Mon Sep 17 00:00:00 2001 From: tamannakakkar93 <51637901+tamannakakkar93@users.noreply.github.com> Date: Thu, 16 May 2024 23:16:43 +0530 Subject: [PATCH 16/16] Update v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../v2/elasticsearch/options/ElasticsearchWriteOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java index a998fa4fe9..b6089753f1 100644 --- a/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java +++ b/v2/elasticsearch-common/src/main/java/com/google/cloud/teleport/v2/elasticsearch/options/ElasticsearchWriteOptions.java @@ -161,7 +161,7 @@ public interface ElasticsearchWriteOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to JavaScript UDF source for _id metadata", helpText = - "The Cloud Storage path to the JavaScript UDF source for the function that specifies `_id` metadata be include with the document in bulk requests. Defaults to: none.") + "The Cloud Storage path to the JavaScript UDF source for the function that specifies `_id` metadata to include with the document in bulk requests. Defaults to: none.") String getJavaScriptIdFnGcsPath(); void setJavaScriptIdFnGcsPath(String javaScriptTextTransformGcsPath);