Skip to content

Commit

Permalink
Modified parameters for Stream_GCS_Text_to_BigQuery_Flex template
Browse files Browse the repository at this point in the history
  • Loading branch information
rajkgupt committed May 3, 2024
1 parent 45a5b7f commit ad65fdd
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 19 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,9 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions {
optional = true,
description = "Cloud Storage path to Javascript UDF source",
helpText =
"The Cloud Storage path pattern for the JavaScript code containing your user-defined "
+ "functions.",
example = "gs://your-bucket/your-function.js")
"\tThe Cloud Storage URI of the .js file that defines the JavaScript user-defined function (UDF) "
+ "you want to use.",
example = "gs://my-bucket/my-udfs/my_file.js")
String getJavascriptTextTransformGcsPath();

void setJavascriptTextTransformGcsPath(String javascriptTextTransformGcsPath);
Expand All @@ -102,7 +102,8 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions {
optional = true,
description = "JavaScript UDF auto-reload interval (minutes)",
helpText =
"Define the interval that workers may check for JavaScript UDF changes to reload the files.")
"Define the interval that workers may check for JavaScript UDF changes to reload the files. Defaults " +
"to: 0.")
@Default.Integer(0)
Integer getJavascriptTextTransformReloadIntervalMinutes();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,8 +215,8 @@ public interface Options
optional = false,
regexes = {"^gs:\\/\\/[^\\n\\r]+$"},
description = "The GCS location of the text you'd like to process",
helpText = "The path to the Cloud Storage text to read.",
example = "gs://your-bucket/your-file.txt")
helpText = "Cloud Storage location of the text you'd like to process.",
example = "gs://my-bucket/my-files/text.txt")
String getInputFilePattern();

void setInputFilePattern(String value);
Expand All @@ -225,8 +225,8 @@ public interface Options
order = 2,
optional = false,
description = "JSON file with BigQuery Schema description",
helpText = "The Cloud Storage path to the JSON file that defines your BigQuery schema.",
example = "gs://your-bucket/your-schema.json")
helpText = "Cloud Storage location of your BigQuery schema file, described as a JSON.",
example = "gs://path/to/my/schema.json")
String getJSONPath();

void setJSONPath(String value);
Expand All @@ -237,9 +237,8 @@ public interface Options
regexes = {".+:.+\\..+"},
description = "Output table to write to",
helpText =
"The location of the BigQuery table in which to store your processed data. If you reuse"
+ " an existing table, it will be overwritten.",
example = "your-project:your-dataset.your-table")
"The fully qualified BigQuery table.",
example = "my-project:dataset.table")
String getOutputTable();

void setOutputTable(String value);
Expand All @@ -263,8 +262,10 @@ public interface Options
regexes = {"[a-zA-Z0-9_]+"},
description = "UDF Javascript Function Name",
helpText =
"The name of the function to call from your JavaScript file. Use only letters, digits,"
+ " and underscores.",
"The name of the JavaScript user-defined function (UDF) that you want to use. For example, if your " +
"JavaScript function code is myTransform(inJson) { /*...do stuff...*/ }, then the function name " +
"is myTransform. For sample JavaScript UDFs, see UDF Examples " +
"(https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples)",
example = "transform_udf1")
String getJavascriptTextTransformFunctionName();

Expand All @@ -275,8 +276,8 @@ public interface Options
order = 6,
optional = false,
description = "Temporary directory for BigQuery loading process",
helpText = "Temporary directory for the BigQuery loading process.",
example = "gs://your-bucket/your-files/temp-dir")
helpText = "Temporary directory for BigQuery loading process.",
example = "gs://my-bucket/my-files/temp_dir")
String getBigQueryLoadingTemporaryDirectory();

void setBigQueryLoadingTemporaryDirectory(String directory);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -479,10 +479,9 @@ public interface TextToBigQueryStreamingOptions
optional = true,
description = "The dead-letter table name to output failed messages to BigQuery",
helpText =
"BigQuery table for failed messages. Messages failed to reach the output table for different reasons "
+ "(e.g., mismatched schema, malformed json) are written to this table. If it doesn't exist, it will"
+ " be created during pipeline execution. If not specified, \"outputTableSpec_error_records\" is used instead.",
example = "your-project-id:your-dataset.your-table-name")
"Table for messages that failed to reach the output table. For example: " +
"my-project:dataset.my-unprocessed-table. If it doesn't exist, it is created during pipeline " +
"execution. If not specified, <outputTableSpec>_error_records is used instead.")
String getOutputDeadletterTable();

void setOutputDeadletterTable(String value);
Expand Down

0 comments on commit ad65fdd

Please sign in to comment.