Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add JAVASCRIPT_UDF_FILE type to template parameter type #1528

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,41 @@ public final class TemplateParameter {
boolean hiddenUi() default false;
}

/** Template parameter containing a JavaScript UDF in Cloud Storage. */
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.METHOD})
public @interface JavascriptUdfFile {
/** Order of appearance. */
int order() default 999;

/** Name of the parameter. */
String name() default "";

/** Group Name of the parameter. */
String groupName() default "";

/** Parent Name of the parameter. */
String parentName() default "";

/** List of parent trigger values. */
String[] parentTriggerValues() default "";

/** If parameter is optional. */
boolean optional() default false;

/** Description of the parameter. */
String description();

/** Help text of the parameter. */
String helpText();

/** Example of the parameter. */
String example() default "";

/** Parameter visibility in the UI. */
boolean hiddenUi() default false;
}

/** Template Parameter containing a BigQuery table to read/write. */
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.METHOD})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public final class MetadataUtils {
TemplateParameter.GcsWriteFile.class,
TemplateParameter.GcsWriteFolder.class,
TemplateParameter.Integer.class,
TemplateParameter.JavascriptUdfFile.class,
TemplateParameter.KafkaTopic.class,
TemplateParameter.KmsEncryptionKey.class,
TemplateParameter.Long.class,
Expand Down Expand Up @@ -129,6 +130,10 @@ public static List<String> getRegexes(Annotation parameterAnnotation) {
return Arrays.asList(simpleTextParam.regexes());
}
return null;
case "JavascriptUdfFile":
TemplateParameter.JavascriptUdfFile javascriptUdfFileParam =
(TemplateParameter.JavascriptUdfFile) parameterAnnotation;
return List.of("^gs:\\/\\/[^\\n\\r]+$");
case "GcsReadFile":
TemplateParameter.GcsReadFile gcsReadFileParam =
(TemplateParameter.GcsReadFile) parameterAnnotation;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,23 @@ public void processParamType(Annotation parameterAnnotation) {
this.setHiddenUi(simpleTextParam.hiddenUi());
this.setParamType(ImageSpecParameterType.TEXT);

break;
case "JavascriptUdfFile":
TemplateParameter.JavascriptUdfFile javascriptUdfFileParam =
(TemplateParameter.JavascriptUdfFile) parameterAnnotation;
if (!javascriptUdfFileParam.name().isEmpty()) {
this.setName(javascriptUdfFileParam.name());
}
processDescriptions(
javascriptUdfFileParam.groupName(),
javascriptUdfFileParam.description(),
javascriptUdfFileParam.helpText(),
javascriptUdfFileParam.example());
this.setParentName(javascriptUdfFileParam.parentName());
this.setParentTriggerValues(javascriptUdfFileParam.parentTriggerValues());
this.setOptional(javascriptUdfFileParam.optional());
this.setHiddenUi(javascriptUdfFileParam.hiddenUi());
this.setParamType(ImageSpecParameterType.JAVASCRIPT_UDF_FILE);
break;
case "GcsReadFile":
TemplateParameter.GcsReadFile gcsReadFileParam =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ public enum ImageSpecParameterType {
/** BigQuery table reference. */
BIGQUERY_TABLE,

/** JavaScript UDF in Cloud Storage. */
JAVASCRIPT_UDF_FILE,

/** Cloud Storage glob to read file(s). */
GCS_READ_FILE,

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ public void testSampleAtoBOk() {
ImageSpecParameter inputKafkaTopic = metadata.getParameter("inputKafkaTopic").get();
assertEquals(ImageSpecParameterType.KAFKA_TOPIC, inputKafkaTopic.getParamType());

ImageSpecParameter javascriptUdfFile = metadata.getParameter("javascriptUdfFile").get();
assertEquals(ImageSpecParameterType.JAVASCRIPT_UDF_FILE, javascriptUdfFile.getParamType());

ImageSpecParameter logical = metadata.getParameter("logical").get();
assertEquals(ImageSpecParameterType.BOOLEAN, logical.getParamType());
assertEquals("^(true|false)$", logical.getRegexes().get(0));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ public interface AtoBOptions {
@Default.Boolean(false)
Boolean getParamWithGroupName();

@TemplateParameter.JavascriptUdfFile(
order = 16,
description = "JavaScript UDF in Cloud Storage",
helpText = "JavaScript UDF in Cloud Storage")
@Default.String("")
String javascriptUdfFile();

@TemplateParameter.Text(
order = 9,
parentName = "paramWithGroupName",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public abstract class JavascriptTextTransformer {

/** Necessary CLI options for running UDF function. */
public interface JavascriptTextTransformerOptions extends PipelineOptions {
@TemplateParameter.GcsReadFile(
@TemplateParameter.JavascriptUdfFile(
order = 1,
optional = true,
description = "Cloud Storage path to Javascript UDF source",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ public interface Options

void setOutputTable(String value);

@TemplateParameter.GcsReadFile(
@TemplateParameter.JavascriptUdfFile(
order = 4,
optional = false,
description = "GCS path to javascript fn for transforming output",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public interface BigQueryWriteOptions extends PipelineOptions, DataflowPipelineO

/** UDF options. */
public interface JavascriptDocumentTransformerOptions extends PipelineOptions {
@TemplateParameter.GcsReadFile(
@TemplateParameter.JavascriptUdfFile(
order = 1,
optional = true,
description = "JavaScript UDF path in Cloud Storage.",
Expand Down