diff --git a/.gitignore b/.gitignore index 3af0892..649b4e5 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,10 @@ release.properties # Remove dev directory. dev +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +*.xml +# project files +*.iml +.idea \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml index 7461fb6..40aface 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -1,13 +1,10 @@ + + "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" + "https://checkstyle.org/dtds/configuration_1_3.dtd"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - + + - - - - - - - - - - + + + + - - - - + + + + - - - + + + - - - - - + + + + - - MODIFIERS CHECKS + + + + + + + + + + + + + + + + + + + + + + + + + - --> + + + + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - WHITESPACE CHECKS + + + + + + + + + + + - --> - + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + - - - + + + + + - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/docs/SendGrid-batchsink.md b/docs/SendGrid-batchsink.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/SendGrid-batchsource.md b/docs/SendGrid-batchsource.md new file mode 100644 index 0000000..d7f72f8 --- /dev/null +++ b/docs/SendGrid-batchsource.md @@ -0,0 +1,64 @@ +# SendGrid batch source + +Description +----------- +Plugin fetches data from SendGrid. SendGrid is a cloud-based service that assists businesses with email delivery. For +the end user SendGrid provides information about existing Marketing Campaigns, Email Analytics, Bounces, Spam Reports. + +Properties +---------- +### General + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Authentication type:** The way, how user would like to be authenticated to the SendGrid account + +**API Key:** The SendGrid API Key taken from the SendGrid account + +**Username:** Login name for the SendGrid account + +**Password:** Password for the SendGrid account + +**Data Source Types:** List of data source groups + +Available: +- Marketing Campaigns Fields +- Stats Fields +- Suppressions Fields + +**Data Source:** SendGrid source object + +Available: +- Marketing Campaigns Fields + - Automation + - Single Sends + - Senders + - Contacts + - Segments +- Stats + - Global Stats + - Category Stats + - Advanced Stats +- Suppressions + - Bounces + - Global Unsubscribes + - Group Unsubscribes + +**Data Source Fields:** The list of fields available for the retrieval + +- *Automation:* id, name, status, type, message_count, created_at, updated_at, live_at +- *SingleSends:* id, name, status, created_at, updated_at, +- *Senders:* id, nickname, address, address_2, city, country, state, zip, locked, created_at, updated_at, from(email, name), verified(status, reason), reply_to(name, email) +- *Contacts:* id, first_name, last_name, list_ids, created_at, updated_at, email, Segments, id, name, parent_list_id, created_at, updated_at, sample_updated_at, contacts_count +- *GlobalStats:* date ,blocks ,bounce_drops ,bounces ,clicks ,deferred ,invalid_emails ,opens ,processed ,requests ,spam_report_drops ,spam_reports ,unique_clicks ,unique_opens ,unsubscribe_drops ,unsubscribes +- *CategoryStats:* name, type, date, blocks, bounce_drops, bounces, clicks, deferred, delivered, invalid_emails, opens, processed, requests, spam_report_drops, spam_reports, unique_clicks, unique_opens, unsubscribe_drops, unsubscribes +- *AdvancedStats:* name, type, date, clicks, opens, unique_clicks, unique_opens +- *Bounces:* created, email, reason, status +- *GroupUnsubscribes:* id ,name ,description ,is_default ,last_email_send_at ,unsubscribes + +**Start Date:** The date in format YYYY-MM-DD, starting from which the data is requested + +**End Date:** The date in format YYYY-MM-DD, the end date for the requested data + + + diff --git a/icons/SendGrid-batchsink.png b/icons/SendGrid-batchsink.png new file mode 100644 index 0000000..a716ff7 Binary files /dev/null and b/icons/SendGrid-batchsink.png differ diff --git a/icons/SendGrid-batchsource.png b/icons/SendGrid-batchsource.png new file mode 100644 index 0000000..a716ff7 Binary files /dev/null and b/icons/SendGrid-batchsource.png differ diff --git a/pom.xml b/pom.xml index f5cf210..7d71c9b 100644 --- a/pom.xml +++ b/pom.xml @@ -1,6 +1,6 @@ - - - 4.0.0 - - io.cdap.plugin - sendgrid - 1.1.0-SNAPSHOT - sendgrid-plugins - jar - Sendgrid Plugins - - - UTF-8 - 6.1.1 - 2.2.0 - 1.4.1 - ${project.basedir} - - - - - CDAP - cdap-dev@googlegroups.com - CDAP - http://cdap.io - - + 4.0.0 - - scm:git:https://github.com/data-integrations/sendgrid.git - scm:git@github.com:data-integrations/sendgrid.git - https://github.com/data-integrations/sendgrid - HEAD - + io.cdap.plugin + sendgrid + 1.3.0-SNAPSHOT + jar + SendGrid plugin - - https://issues.cask.co/browse/CDAP - + + UTF-8 + 6.1.0-SNAPSHOT + 2.8.0 + 2.3.0-SNAPSHOT + 2.2.4 + 1.2 + 4.11 + 1.10.19 + 2.1.3 + - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - A business-friendly OSS license - - + + scm:git:https://github.com/data-integrations/sendgrid.git + scm:git@github.com:data-integrations/sendgrid.git + https://github.com/data-integrations/sendgrid + HEAD + - - - sonatype - https://oss.sonatype.org/content/groups/public - - - sonatype-snapshots - https://oss.sonatype.org/content/repositories/snapshots - - + + https://issues.cask.co/browse/CDAP + - - - com.sendgrid - sendgrid-java - 4.2.0 - - - com.sendgrid - java-http-client - 4.2.0 - - - io.cdap.cdap - cdap-etl-api - ${cdap.version} - provided - - - io.cdap.plugin - hydrator-common - ${cdap.plugin.version} - - - javax.mail - mail - ${javamail.version} - - + + + sonatype + https://oss.sonatype.org/content/groups/public + + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots + + + + + sonatype + https://oss.sonatype.org/content/groups/public/ + + - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - - - - org.apache.felix - maven-bundle-plugin - 3.5.1 - true - - - *;inline=false;scope=compile - true - lib - - <_exportcontents>io.cdap.plugin.sendgrid.*; - - - - - package - - bundle - - - - - - io.cdap - cdap-maven-plugin - 1.1.0 - - - system:cdap-data-pipeline[6.0.0-SNAPSHOT,7.0.0-SNAPSHOT) - system:cdap-data-streams[6.0.0-SNAPSHOT,7.0.0-SNAPSHOT) - - - - - create-artifact-config - prepare-package - - create-plugin-json - - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.14.1 - - - org.apache.felix - maven-bundle-plugin - - - io.cdap - cdap-maven-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - 2.17 - - - validate - process-test-classes - - checkstyle.xml - suppressions.xml - UTF-8 - true - true - true - - - check - - - - - - com.puppycrawl.tools - checkstyle - 6.19 - - - - - + + + commons-logging + commons-logging + ${common.logging.version} + compile + + + com.sendgrid + sendgrid-java + 4.4.1 + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + + + org.apache.httpcomponents + httpclient + 4.5.2 + compile + + + org.apache.httpcomponents + httpcore + 4.4.4 + compile + + + + com.google.code.gson + gson + ${gson.version} + compile + + + io.cdap.cdap + cdap-etl-api + ${cdap.version} + provided + + + com.google.code.gson + gson + + + + + io.cdap.plugin + hydrator-common + ${hydrator.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.avro + avro + + + org.apache.zookeeper + zookeeper + + + com.google.guava + guava + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + servlet-api + javax.servlet + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + jasper-compiler + tomcat + + + jasper-runtime + tomcat + + + jsp-api + javax.servlet.jsp + + + slf4j-api + org.slf4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + org.slf4j + slf4j-log4j12 + + + com.google.inject.extensions + guice-servlet + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + + + io.cdap.cdap + hydrator-test + ${cdap.version} + test + + + junit + junit + ${junit.version} + test + + + io.cdap.cdap + cdap-data-pipeline + ${cdap.version} + test + + + org.mockito + mockito-all + ${mockito.version} + test + + + com.google.inject + guice + 4.2.2 + test + + + + + + org.apache.felix + maven-bundle-plugin + 3.3.0 + true + + + <_exportcontents>io.cdap.plugin.sendgrid.* + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + io.cdap + cdap-maven-plugin + 1.1.0 + + + system:cdap-data-pipeline[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT) + + + + + create-artifact-config + prepare-package + + create-plugin-json + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + process-test-classes + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + **/org/apache/cassandra/**,**/org/apache/hadoop/** + + + check + + + + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.14.1 + + -Xmx5000m -Djava.awt.headless=true -XX:+UseG1GC -XX:OnOutOfMemoryError="kill -9 %p" -Djava.net.preferIPv4Stack=true + false + plain + + ${project.build.directory} + + + **/*TestsSuite.java + **/*TestSuite.java + **/Test*.java + **/*Test.java + **/*TestCase.java + + + **/*TestRun.java + + + + + diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormat.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormat.java new file mode 100644 index 0000000..f406fcc --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormat.java @@ -0,0 +1,72 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMail; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; + +/** + * An OutputFormat that sends the output of a Hadoop job to the SendGrid record writer, also + * it defines the output committer. + */ +public class SendGridOutputFormat extends OutputFormat { + @Override + public RecordWriter getRecordWriter(TaskAttemptContext taskAttemptContext) { + return new SendGridRecordWriter(taskAttemptContext); + } + + @Override + public void checkOutputSpecs(JobContext jobContext) throws IOException, InterruptedException { + // no-op + } + + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) { + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) throws IOException { + + } + + @Override + public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException { + + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException { + return false; + } + + @Override + public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException { + + } + + @Override + public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException { + + } + }; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormatProvider.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormatProvider.java new file mode 100644 index 0000000..86e56e3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormatProvider.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.OutputFormatProvider; + +import java.util.Map; + +/** + * Provides {@link SendGridOutputFormat} class name and configuration + */ +public class SendGridOutputFormatProvider implements OutputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.sendgrid.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + SendGridOutputFormatProvider(SendGridSinkConfig config) { + this.conf = new ImmutableMap.Builder() + .put(PROPERTY_CONFIG_JSON, gson.toJson(config)) + .build(); + } + + @Override + public String getOutputFormatClassName() { + return SendGridOutputFormat.class.getName(); + } + + @Override + public Map getOutputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridRecordWriter.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridRecordWriter.java new file mode 100644 index 0000000..23b97e0 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridRecordWriter.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMail; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + + +import java.io.IOException; + +/** + * Writes {@link SendGridMail} into batches and submit them to SendGrid send API + */ +public class SendGridRecordWriter extends RecordWriter { + private static final Gson gson = new GsonBuilder().create(); + private SendGridClient client; + + public SendGridRecordWriter(TaskAttemptContext taskAttemptContext) { + Configuration conf = taskAttemptContext.getConfiguration(); + String serializedConfig = conf.get(SendGridOutputFormatProvider.PROPERTY_CONFIG_JSON); + SendGridSinkConfig sgConfig = gson.fromJson(serializedConfig, SendGridSinkConfig.class); + + client = new SendGridClient(sgConfig); + } + + @Override + public void write(NullWritable nullWritable, SendGridMail sendGridMail) throws IOException { + client.sendMail(sendGridMail); + } + + @Override + public void close(TaskAttemptContext taskAttemptContext) { + // no-op + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSink.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSink.java new file mode 100644 index 0000000..328a552 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSink.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Output; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.cdap.etl.api.batch.BatchSinkContext; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; +import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMail; +import org.apache.hadoop.io.NullWritable; + +import java.util.stream.Collectors; + +/** + * Batch Sink Plugin + */ +@Plugin(type = BatchSink.PLUGIN_TYPE) +@Name(BaseConfig.PLUGIN_NAME) +@Description("Sends mails via SendGrid") +public class SendGridSink extends BatchSink { + + private final SendGridSinkConfig config; + + public SendGridSink(SendGridSinkConfig config) { + this.config = config; + } + + @Override + @SuppressWarnings("ThrowableNotThrown") + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); + + IdUtils.validateReferenceName(config.referenceName, failureCollector); + + config.validate(failureCollector); + config.validate(pipelineConfigurer.getStageConfigurer().getInputSchema()); + + failureCollector.getOrThrowException(); + } + + @Override + public void prepareRun(BatchSinkContext batchSinkContext) { + Schema inputSchema = batchSinkContext.getInputSchema(); + config.validate(inputSchema); + + batchSinkContext.addOutput(Output.of(config.referenceName, new SendGridOutputFormatProvider(config))); + + LineageRecorder lineageRecorder = new LineageRecorder(batchSinkContext, config.referenceName); + lineageRecorder.createExternalDataset(inputSchema); + + if (inputSchema.getFields() != null && !inputSchema.getFields().isEmpty()) { + String operationDescription = String.format("Wrote to SendGrid %s", config.getFrom()); + lineageRecorder.recordWrite("Write", operationDescription, + inputSchema.getFields().stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + } + } + + @Override + public void transform(StructuredRecord record, Emitter> emitter) { + SendGridMail sendGridMail = SendGridSinkTransformer.transform(config, record); + emitter.emit(new KeyValue<>(null, sendGridMail)); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java new file mode 100644 index 0000000..777f6dc --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java @@ -0,0 +1,255 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * SendGrid Sink Plugin configuration + */ +public class SendGridSinkConfig extends BaseConfig { + public static final String PROPERTY_RECIPIENT_ADDRESS_SOURCE = "recipientAddressSource"; + public static final String PROPERTY_RECIPIENT_CONFIG_ADDRESS = "recipientConfigAddressList"; + public static final String PROPERTY_RECIPIENT_COLUMN = "recipientColumnName"; + public static final String PROPERTY_BODY_COLUMN = "bodyColumnName"; + public static final String PROPERTY_FROM = "from"; + public static final String PROPERTY_REPLY_TO = "replyTo"; + public static final String PROPERTY_FOOTER_ENABLED = "footerEnabled"; + public static final String PROPERTY_FOOTER_HTML = "footerHtml"; + public static final String PROPERTY_SANDBOX_MODE = "sandboxMode"; + public static final String PROPERTY_CLICK_TRACKING = "clickTracking"; + public static final String PROPERTY_OPEN_TRACKING = "openTracking"; + public static final String PROPERTY_SUBSCRIPTION_TRACKING = "subscriptionTracking"; + public static final String PROPERTY_MAIL_SUBJECT = "mailSubject"; + + public static final String TO_TYPE_INPUT = "input"; + public static final String TO_TYPE_CONFIG = "config"; + + public String getMailSubject() { + return mailSubject; + } + + /** + * Available sources for recipient addresses + */ + public enum ToAddressSource { + CONFIG, + INPUT; + + public static ToAddressSource fromString(String toType) { + switch (toType) { + case TO_TYPE_INPUT: + return ToAddressSource.INPUT; + case TO_TYPE_CONFIG: + return ToAddressSource.CONFIG; + default: + throw new IllegalArgumentException(String.format("Unknown address source '%s', allowed: '%s', '%s'", + toType, TO_TYPE_INPUT, TO_TYPE_CONFIG)); + } + } + } + + @Name(PROPERTY_FROM) + @Description("The author of the message") + @Macro + private String from; + + @Name(PROPERTY_REPLY_TO) + @Description("Email address to which the author of the message suggests that replies be sent") + @Nullable + @Macro + private String replyTo; + + @Name(PROPERTY_FOOTER_ENABLED) + @Description("Footer feature setting switcher") + @Nullable + @Macro + private String footerEnable; + + @Name(PROPERTY_FOOTER_HTML) + @Description("The default footer which would be included to every email") + @Nullable + @Macro + private String footerHTML; + + @Name(PROPERTY_SANDBOX_MODE) + @Description("Allows to send a test email to ensure that your request body is valid and formatted correctly") + @Nullable + @Macro + private String sandboxMode; + + @Name(PROPERTY_CLICK_TRACKING) + @Description("Allows to track whether a recipient clicked a link in a email") + @Nullable + @Macro + private String clickTracking; + + @Name(PROPERTY_OPEN_TRACKING) + @Description("Allows to track whether the email was opened or not, by including a single pixel image in the" + + " body of the content. When the pixel is loaded, SendGrid can log that the email was opened") + @Nullable + @Macro + private String openTracking; + + @Name(PROPERTY_SUBSCRIPTION_TRACKING) + @Description("Allows to insert a subscription management link at the bottom of the text and html" + + " bodies of an email") + @Nullable + @Macro + private String subscriptionTracking; + + @Name(PROPERTY_RECIPIENT_ADDRESS_SOURCE) + @Description("Recipients addresses source selection") + @Macro + private String recipientAddressSource; + + @Name(PROPERTY_RECIPIENT_CONFIG_ADDRESS) + @Description("List of mail recipients") + @Nullable + @Macro + private String recipientConfigAddressList; + + @Name(PROPERTY_RECIPIENT_COLUMN) + @Description("Name of the column with coma-separated list of recipients") + @Nullable + @Macro + private String recipientColumnName; + + @Name(PROPERTY_BODY_COLUMN) + @Description("Name of the column for the mail content") + @Macro + private String bodyColumnName; + + @Name(PROPERTY_MAIL_SUBJECT) + @Description("Email Subject") + @Macro + private String mailSubject; + + /** + * Constructor + * + * @param referenceName uniquely identify source/sink for lineage, annotating metadata, etc. + */ + public SendGridSinkConfig(String referenceName) { + super(referenceName); + } + + public void validate(FailureCollector failureCollector) { + new SendGridSinkConfigValidator(failureCollector, this).validate(); + } + + private void validateField(Schema.Field field, String name) { + if (field == null) { + throw new IllegalArgumentException(String.format("Plugin is configured to use column '%s' for" + + " recipient addresses, but input schema did not provide such column", recipientColumnName)); + } + + Schema fieldSchema = field.getSchema(); + if (fieldSchema.getType() == Schema.Type.UNION) { + if (fieldSchema.getUnionSchemas().stream().noneMatch(x -> x.getType() == Schema.Type.STRING)) { + throw new IllegalArgumentException(String.format("The input schema column '%s' expected to be of type STRING", + name)); + } + return; + } + + if (fieldSchema.getType() != Schema.Type.STRING) { + throw new IllegalArgumentException(String.format("The input schema column '%s' expected to be of type STRING", + name)); + } + } + + public void validate(Schema schema) { + if (schema == null) { + throw new IllegalArgumentException("Input schema cannot be empty"); + } + + if (getRecipientAddressSource() == ToAddressSource.INPUT) { + validateField(schema.getField(recipientColumnName), recipientColumnName); + } + + validateField(schema.getField(bodyColumnName), bodyColumnName); + } + + public String getFrom() { + if (Strings.isNullOrEmpty(from)) { + throw new IllegalArgumentException(String.format("Property '%s' cannot be empty", PROPERTY_FROM)); + } + return from; + } + + public ToAddressSource getRecipientAddressSource() { + return ToAddressSource.fromString(recipientAddressSource); + } + + public List getRecipientAddresses() { + if (recipientConfigAddressList == null) { + return Collections.emptyList(); + } + return Arrays.asList(recipientConfigAddressList.split(",")); + } + + @Nullable + public String getRecipientColumnName() { + return recipientColumnName; + } + + @Nullable + public String getBodyColumnName() { + return bodyColumnName; + } + + @Nullable + public String getReplyTo() { + return replyTo; + } + + public Boolean getFooterEnable() { + return !Strings.isNullOrEmpty(footerEnable) && footerEnable.equals("true"); + } + + public String getFooterHTML() { + return footerHTML; + } + + public Boolean getSandboxMode() { + return !Strings.isNullOrEmpty(sandboxMode) && sandboxMode.equals("true"); + } + + public Boolean getClickTracking() { + return !Strings.isNullOrEmpty(clickTracking) && clickTracking.equals("true"); + } + + public Boolean getOpenTracking() { + return !Strings.isNullOrEmpty(openTracking) && openTracking.equals("true"); + } + + public Boolean getSubscriptionTracking() { + return !Strings.isNullOrEmpty(subscriptionTracking) && subscriptionTracking.equals("true"); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigValidator.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigValidator.java new file mode 100644 index 0000000..9054fbf --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigValidator.java @@ -0,0 +1,83 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import com.google.common.base.Strings; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.common.config.BaseConfigValidator; + +import java.util.regex.Pattern; + +/** + * Sink Config Validator + */ +public class SendGridSinkConfigValidator extends BaseConfigValidator { + + private SendGridSinkConfig config; + private static Pattern basicMailRegEx = Pattern.compile("^[\\w\\d.+\\-]+@[\\w\\d]+\\.[\\w\\d]+$"); + + public SendGridSinkConfigValidator(FailureCollector failureCollector, SendGridSinkConfig config) { + super(failureCollector, config); + + this.config = config; + } + + /** + * Check if provided string is a valid email + * @param email provided email address + */ + private boolean validateEmail(String email) { + return basicMailRegEx.matcher(email).find(); + } + + private void checkFrom() { + if (Strings.isNullOrEmpty(config.getFrom()) || !validateEmail(config.getFrom())) { + failureCollector.addFailure("Not a valid or empty email address", null) + .withConfigProperty(SendGridSinkConfig.PROPERTY_FROM); + } + } + + private void checkReplyTo() { + if (!Strings.isNullOrEmpty(config.getReplyTo()) && validateEmail(config.getReplyTo())) { + failureCollector.addFailure("Not a valid or empty email address", null) + .withConfigProperty(SendGridSinkConfig.PROPERTY_FROM); + } + } + + private void checkFooter() { + if (config.getFooterEnable() && Strings.isNullOrEmpty(config.getFooterHTML())) { + failureCollector.addFailure("Footer content cannot be empty", null) + .withConfigProperty(SendGridSinkConfig.PROPERTY_FOOTER_HTML); + } + } + + /** + * Perform validation tasks which did not involve API Client usage + */ + @Override + public void doValidation() { + if (!config.containsMacro(SendGridSinkConfig.PROPERTY_FROM)) { + checkFrom(); + } + if (!config.containsMacro(SendGridSinkConfig.PROPERTY_REPLY_TO)) { + checkReplyTo(); + } + if (!config.containsMacro(SendGridSinkConfig.PROPERTY_FOOTER_ENABLED) && + !config.containsMacro(SendGridSinkConfig.PROPERTY_FOOTER_HTML)) { + checkFooter(); + } + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkTransformer.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkTransformer.java new file mode 100644 index 0000000..52907ba --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkTransformer.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMail; +import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMailBuilder; + +import java.util.Arrays; + +/** + * {@link StructuredRecord} to {@link SendGridMail} + */ +public class SendGridSinkTransformer { + + private static Object readRecordField(StructuredRecord record, String fieldName, Schema.Type fieldType) { + Schema.Field field = record.getSchema().getField(fieldName); + if (field == null) { + throw new IllegalArgumentException(String.format("Input schema does not provide column '%s'", fieldName)); + } + Schema fieldSchema = field.getSchema(); + + if (fieldSchema.getType() == Schema.Type.UNION) { // check the situation if field is nullable + if (fieldSchema.getUnionSchemas().stream().noneMatch(x -> x.getType() == fieldType)) { + throw new IllegalArgumentException(String.format("Column '%s' does not belong to type '%s'''", + fieldName, fieldType.name())); + } + } else if (field.getSchema().getType() != fieldType) { + throw new IllegalArgumentException(String.format("Column '%s' does not belong to type '%s'''", + fieldName, fieldType.name())); + } + Object objRecipients = record.get(fieldName); + if (objRecipients == null) { + throw new IllegalArgumentException("Record provided empty list of recipients"); + } + return objRecipients; + } + + public static SendGridMail transform(SendGridSinkConfig config, StructuredRecord record) { + SendGridMailBuilder builder = SendGridMailBuilder.getInstance(); + + builder.from(config.getFrom()); + if (config.getRecipientAddressSource() == SendGridSinkConfig.ToAddressSource.CONFIG) { + config.getRecipientAddresses().forEach(builder::addTo); + } else if (config.getRecipientAddressSource() == SendGridSinkConfig.ToAddressSource.INPUT) { + Object recipientFieldValue = readRecordField(record, config.getRecipientColumnName(), Schema.Type.STRING); + Arrays + .asList(((String) recipientFieldValue).split(",")) + .forEach(builder::addTo); + } + builder.replyTo(config.getReplyTo()); + builder.subject(config.getMailSubject()); + + Object bodyValue = readRecordField(record, config.getBodyColumnName(), Schema.Type.STRING); + builder.addHtmlContent((String) bodyValue); + builder.footerHtml(config.getFooterEnable(), config.getFooterHTML()); + + builder.clickTracking(config.getClickTracking()); + builder.openTracking(config.getOpenTracking()); + builder.subscriptionTracking(config.getSubscriptionTracking()); + builder.sandboxMode(config.getSandboxMode()); + + return builder.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridInputFormat.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridInputFormat.java new file mode 100644 index 0000000..0c81fbd --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridInputFormat.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.util.Collections; +import java.util.List; + +/** + * SendGrid InputFormat + */ +public class SendGridInputFormat extends InputFormat { + private static final Gson gson = new GsonBuilder().create(); + + @Override + public List getSplits(JobContext context) { + return Collections.singletonList(new SendGridSplit()); + } + + @Override + public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) { + + Configuration conf = context.getConfiguration(); + String serializedConfig = conf.get(SendGridInputFormatProvider.PROPERTY_CONFIG_JSON); + SendGridSourceConfig sgConfig = gson.fromJson(serializedConfig, SendGridSourceConfig.class); + + return (sgConfig.isMultiObjectMode()) + ? new SendGridMultiRecordReader() + : new SendGridRecordReader(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridInputFormatProvider.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridInputFormatProvider.java new file mode 100644 index 0000000..455de7a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridInputFormatProvider.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Map; + +/** + * Input stream format provider + */ +public class SendGridInputFormatProvider implements InputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.sendgrid.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + SendGridInputFormatProvider(SendGridSourceConfig config) { + this.conf = new ImmutableMap.Builder() + .put(PROPERTY_CONFIG_JSON, gson.toJson(config)) + .build(); + } + + @Override + public String getInputFormatClassName() { + return SendGridInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridMultiRecordReader.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridMultiRecordReader.java new file mode 100644 index 0000000..f4000da --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridMultiRecordReader.java @@ -0,0 +1,102 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.MultiObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * SendGrid MultiRecord Reader + */ +public class SendGridMultiRecordReader extends RecordReader { + private static final Gson gson = new GsonBuilder().create(); + + private Map> recordIterators; + private Map currentRecords; + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) { + String serializedConfig = context.getConfiguration().get(SendGridInputFormatProvider.PROPERTY_CONFIG_JSON); + SendGridSourceConfig sgConfig = gson.fromJson(serializedConfig, SendGridSourceConfig.class); + SendGridClient client = new SendGridClient(sgConfig); + ImmutableMap.Builder> iterators = new ImmutableMap.Builder<>(); + + sgConfig.getDataSource().forEach(x -> { + ObjectInfo objectInfo = ObjectHelper.getObjectInfo(x); + try { + iterators.put(x, client.getObject(objectInfo, sgConfig.getRequestArguments()).iterator()); + } catch (IOException e) { + iterators.put(x, Collections.emptyIterator()); + } + }); + recordIterators = iterators.build(); + currentRecords = new HashMap<>(); + } + + @Override + public boolean nextKeyValue() { + currentRecords.clear(); + + return recordIterators.entrySet().stream() + .map(k -> { + boolean currHasNext = k.getValue().hasNext(); + if (currHasNext) { + currentRecords.put(k.getKey(), k.getValue().next()); + } + return currHasNext; + }) + .reduce(false, (prev, curr) -> prev || curr); + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public IBaseObject getCurrentValue() { + MultiObject multiObject = new MultiObject(); + currentRecords.forEach(multiObject::addObject); + + return multiObject; + } + + @Override + public float getProgress() { + return 0.0f; + } + + @Override + public void close() { + // no-op + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridRecordReader.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridRecordReader.java new file mode 100644 index 0000000..e97cee8 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridRecordReader.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; + +/** + * SendGrid Record Reader + */ +public class SendGridRecordReader extends RecordReader { + private static final Gson gson = new GsonBuilder().create(); + + private Iterator recordIterator; + private IBaseObject currentRecord; + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) throws IOException { + Configuration conf = context.getConfiguration(); + String serializedConfig = conf.get(SendGridInputFormatProvider.PROPERTY_CONFIG_JSON); + SendGridSourceConfig sgConfig = gson.fromJson(serializedConfig, SendGridSourceConfig.class); + + SendGridClient client = new SendGridClient(sgConfig); + + Iterator objectsIterator = sgConfig.getDataSource().iterator(); + if (objectsIterator.hasNext()) { + ObjectInfo currentObject = ObjectHelper.getObjectInfo(objectsIterator.next()); + recordIterator = client.getObject(currentObject, sgConfig.getRequestArguments()).iterator(); + } else { + recordIterator = Collections.emptyIterator(); + } + } + + @Override + public boolean nextKeyValue() { + boolean recordHasNext = recordIterator.hasNext(); + + if (recordHasNext) { + currentRecord = recordIterator.next(); + } + return recordHasNext; + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public IBaseObject getCurrentValue() { + return currentRecord; + } + + @Override + public float getProgress() { + return 0.0f; + } + + @Override + public void close() { + // no-op + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSource.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSource.java new file mode 100644 index 0000000..acbb1cc --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSource.java @@ -0,0 +1,96 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.common.base.Preconditions; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import org.apache.hadoop.io.NullWritable; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Batch Source plugin + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(BaseConfig.PLUGIN_NAME) +@Description("Reads data from SendGrid API") +public class SendGridSource extends BatchSource { + private final SendGridSourceConfig config; + + public SendGridSource(SendGridSourceConfig config) { + this.config = config; + } + + @Override + public void prepareRun(BatchSourceContext batchSourceContext) { + validateConfiguration(batchSourceContext.getFailureCollector()); + + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading SendGrid Objects", + Preconditions.checkNotNull(config.getSchema().getFields()) + .stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new SendGridInputFormatProvider(config))); + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); + + IdUtils.validateReferenceName(config.referenceName, failureCollector); + validateConfiguration(failureCollector); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void transform(KeyValue input, Emitter emitter) { + Schema schema; + if (config.isMultiObjectMode()) { + List fetchedDataSources = new ArrayList<>(input.getValue().asMap().keySet()); + schema = config.getSchema(fetchedDataSources); + } else { + schema = config.getSchema(); + } + emitter.emit(SendGridSourceTransformer.transform(input.getValue(), schema)); + } + + @SuppressWarnings("ThrowableNotThrown") + private void validateConfiguration(FailureCollector failureCollector) { + config.validate(failureCollector); + failureCollector.getOrThrowException(); + } + +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java new file mode 100644 index 0000000..2240124 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java @@ -0,0 +1,217 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * SendGrid Source Plugin configuration + */ +public class SendGridSourceConfig extends BaseConfig { + public static final String PROPERTY_DATA_SOURCE_TYPES = "dataSourceTypes"; + public static final String PROPERTY_DATA_SOURCE = "dataSource"; + public static final String PROPERTY_DATA_SOURCE_MARKETING = PROPERTY_DATA_SOURCE + "Marketing"; + public static final String PROPERTY_DATA_SOURCE_STATS = PROPERTY_DATA_SOURCE + "Stats"; + public static final String PROPERTY_DATA_SOURCE_SUPPRESSIONS = PROPERTY_DATA_SOURCE + "Suppressions"; + + public static final String PROPERTY_DATA_SOURCE_FIELDS = "dataSourceFields"; + public static final String PROPERTY_STAT_CATEGORIES = "statCategories"; + public static final String PROPERTY_START_DATE = "start_date"; + public static final String PROPERTY_END_DATE = "end_date"; + + @Name(PROPERTY_DATA_SOURCE_TYPES) + @Description("List of data source groups") + @Macro + private String dataSourceTypes; + + @Name(PROPERTY_DATA_SOURCE_MARKETING) + @Description("SendGrid source objects for the Marketing group") + @Macro + @Nullable + private String dataSourceMarketing; + + @Name(PROPERTY_DATA_SOURCE_STATS) + @Description("SendGrid source objects for the Statistics group") + @Macro + @Nullable + private String dataSourceStats; + + @Name(PROPERTY_DATA_SOURCE_SUPPRESSIONS) + @Description("SendGrid source objects for the Suppressions group") + @Macro + @Nullable + private String dataSourceSuppressions; + + @Name(PROPERTY_DATA_SOURCE_FIELDS) + @Description("The list of fields available for the retrieval") + @Macro + @Nullable + private String dataSourceFields; + + @Name(PROPERTY_START_DATE) + @Description("The date in format YYYY-MM-DD, starting from which the data is requested") + @Nullable + @Macro + private String startDate; + + @Name(PROPERTY_END_DATE) + @Description("The date in format YYYY-MM-DD, the end date for the requested data") + @Nullable + @Macro + private String endDate; + + @Name(PROPERTY_STAT_CATEGORIES) + @Description("List of requested categories for the CategoryStats request") + @Nullable + @Macro + private String statCategories; + + private transient Schema schema; + private transient List dataSource; + private transient Boolean multiObjectMode; + + /** + * Constructor + * + * @param referenceName uniquely identify source/sink for lineage, annotating metadata, etc. + */ + public SendGridSourceConfig(String referenceName) { + super(referenceName); + } + + public void validate(FailureCollector failureCollector) { + new SendGridSourceConfigValidator(failureCollector, this).validate(); + } + + /** + * Fetches all fields selected by the user + */ + public List getFields() { + if (Strings.isNullOrEmpty(dataSourceFields)) { + return Collections.emptyList(); + } + + return Arrays.asList(dataSourceFields.split(",")); + } + + /** + * Aggregates categorized data source + */ + public List getDataSource() { + if (dataSource == null) { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + + if (!Strings.isNullOrEmpty(dataSourceMarketing)) { + builder.add(dataSourceMarketing); + } + if (!Strings.isNullOrEmpty(dataSourceStats)) { + builder.add(dataSourceStats); + } + if (!Strings.isNullOrEmpty(dataSourceSuppressions)) { + builder.add(dataSourceSuppressions); + } + + dataSource = Arrays.asList(String.join(",", builder.build()).split(",")); + } + return dataSource; + } + + /** + * Plugin work mode + */ + public boolean isMultiObjectMode() { + if (multiObjectMode == null) { + multiObjectMode = getDataSource().size() > 1; + } + return multiObjectMode; + } + + /** + * Generated schema according to user configuration + * + * @return user configured schema + */ + public Schema getSchema() { + if (schema == null) { + schema = ObjectHelper.buildSchema(getDataSource(), getFields()); + } + return schema; + } + + /** + * Generates limited schema for mentioned in {@code dataSource} sources + * + * @param dataSource sources to be added to the schema + * + * @return custom schema + */ + public Schema getSchema(List dataSource) { + return ObjectHelper.buildSchema(dataSource, getFields(), isMultiObjectMode()); + } + + /** + * Returns query properties required for some SendGrid Objects + * marked with {@link ObjectDefinition#RequiredArguments()} + */ + public Map getRequestArguments() { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + + if (!Strings.isNullOrEmpty(startDate)) { + builder.put(PROPERTY_START_DATE, startDate); + } + if (!Strings.isNullOrEmpty(endDate)) { + builder.put(PROPERTY_END_DATE, endDate); + } + if (!Strings.isNullOrEmpty(statCategories)) { + builder.put(PROPERTY_STAT_CATEGORIES, statCategories); + } + return builder.build(); + } + + @Nullable + public String getStartDate() { + return startDate; + } + + @Nullable + public String getEndDate() { + return endDate; + } + + public List getDataSourceTypes() { + if (!Strings.isNullOrEmpty(dataSourceTypes)) { + return Arrays.asList(dataSourceTypes.split(",")); + } + return Collections.emptyList(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigValidator.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigValidator.java new file mode 100644 index 0000000..7dd48ba --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigValidator.java @@ -0,0 +1,167 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.common.base.Strings; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.batch.sink.SendGridSinkConfig; +import io.cdap.plugin.sendgrid.common.config.BaseConfigValidator; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Validates configuration + */ +public class SendGridSourceConfigValidator extends BaseConfigValidator { + private static Pattern datePattern = Pattern.compile("(?\\d{4})-(?\\d{2})-(?\\d{1,2})"); + private SendGridSourceConfig config; + + public SendGridSourceConfigValidator(FailureCollector failureCollector, SendGridSourceConfig config) { + super(failureCollector, config); + this.config = config; + } + + private Stream getObjectsStream() { + return config.getDataSource().stream() + .filter(x -> !Strings.isNullOrEmpty(x)) + .map(ObjectHelper::getObjectInfo); + } + + private void checkCategoriesSelection() { + if (config.getDataSourceTypes().isEmpty()) { + failureCollector.addFailure("Object categories are not set", null) + .withConfigProperty(SendGridSourceConfig.PROPERTY_DATA_SOURCE_TYPES); + } + + config.getDataSourceTypes() + .forEach(x -> { + try { + DataSourceGroupType.fromString(x); + } catch (IllegalStateException e) { + failureCollector.addFailure( + String.format("Unknown '%s' data source type: %s", x , e.getMessage()), null) + .withStacktrace(e.getStackTrace()); + } + }); + } + + private void checkObjectsSelection() { + List objects = getObjectsStream().collect(Collectors.toList()); + List categories = config.getDataSourceTypes().stream() + .map(DataSourceGroupType::fromString) + .collect(Collectors.toList()); + + categories.forEach(category -> { + if (objects.stream().noneMatch(x -> x.getDataSourceGroupType() == category)) { + failureCollector.addFailure( + String.format("No objects selected for the category: %s", category.name()), null) + .withConfigProperty(SendGridSourceConfig.PROPERTY_DATA_SOURCE); + } + }); + } + + private void checkFieldSelection() { + List objects = getObjectsStream().collect(Collectors.toList()); + List fields = config.getFields(); + + objects.forEach(object -> { + if (object.getFieldsDefinitions(fields).isEmpty()) { + failureCollector.addFailure( + String.format("No fields selected for object '%s'", object.getCdapObjectName()), null) + .withConfigProperty(SendGridSourceConfig.PROPERTY_DATA_SOURCE_FIELDS); + } + }); + } + + private void checkRequiredInputProperties() { + getObjectsStream() + .flatMap(obj -> obj.getRequiredArguments().stream()) + .filter(arg -> !config.getRequestArguments().containsKey(arg)) + .forEach(arg -> failureCollector.addFailure( + String.format("Argument %s cannot be empty", arg), null) + .withConfigProperty(arg)); + } + + private void checkDateFormat(String date, String field) { + if (!Strings.isNullOrEmpty(date)) { + Matcher matcher = datePattern.matcher(date); + if (!matcher.find()) { + failureCollector.addFailure("Input format should math YYYY-MM-DD", null) + .withConfigProperty(field); + } else { + Integer month; + Integer day; + + try { + month = Integer.valueOf(matcher.group("month")); + day = Integer.valueOf(matcher.group("day")); + } catch (NumberFormatException e) { + failureCollector.addFailure("Input format should math YYYY-MM-DD", null) + .withConfigProperty(field); + return; + } + + if (month < 1 || month > 12) { + failureCollector.addFailure( + "Input format should math YYYY-MM-DD and MM should be in range from 1 to 12", null) + .withConfigProperty(field); + } + + if (day < 1 || day > 31) { + failureCollector.addFailure( + "Input format should math YYYY-MM-DD and DD should be in range from 1 to 31", null) + .withConfigProperty(field); + } + } + } + } + + private void checkDateArguments() { + checkDateFormat(config.getStartDate(), SendGridSourceConfig.PROPERTY_START_DATE); + checkDateFormat(config.getEndDate(), SendGridSourceConfig.PROPERTY_END_DATE); + } + + @Override + public void doValidation() { + if (!config.containsMacro(SendGridSourceConfig.PROPERTY_DATA_SOURCE_TYPES)) { + checkCategoriesSelection(); + } + if (!config.containsMacro(SendGridSourceConfig.PROPERTY_DATA_SOURCE)) { + checkObjectsSelection(); + } + if (!config.containsMacro(SendGridSourceConfig.PROPERTY_DATA_SOURCE_FIELDS)) { + checkFieldSelection(); + } + if (!config.containsMacro(SendGridSourceConfig.PROPERTY_STAT_CATEGORIES) && + !config.containsMacro(SendGridSourceConfig.PROPERTY_START_DATE) && + !config.containsMacro(SendGridSourceConfig.PROPERTY_END_DATE)) { + + checkRequiredInputProperties(); + } + + if (!config.containsMacro(SendGridSourceConfig.PROPERTY_START_DATE) && + !config.containsMacro(SendGridSourceConfig.PROPERTY_END_DATE)) { + checkDateArguments(); + } + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceTransformer.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceTransformer.java new file mode 100644 index 0000000..ecd76bf --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceTransformer.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.EmptyObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * {@link IBaseObject} to {@link StructuredRecord} transformer + */ +public class SendGridSourceTransformer { + + @SuppressWarnings("unchecked") + private static void transformValue(String k, Object v, Schema schema, StructuredRecord.Builder builder) { + + if (v instanceof Map) { + Schema mapSchema = Objects.requireNonNull(schema.getField(k)).getSchema(); + builder.set(k, transform((Map) v, mapSchema)); + } else if (v instanceof EmptyObject) { + // no-op + } else if (v instanceof IBaseObject) { + Schema mapSchema = Objects.requireNonNull(schema.getField(k)).getSchema(); + builder.set(k, transform((IBaseObject) v, mapSchema)); + } else if (v instanceof List) { + Schema componentSchema = Objects.requireNonNull(schema.getField(k)).getSchema().getComponentSchema(); + if (componentSchema == null) { + throw new IllegalArgumentException(String.format("Unable to extract schema for the field '%s'", k)); + } + Object values = ((List) v).stream() + .map(arrItem -> transform((Map) arrItem, componentSchema)).collect(Collectors.toList()); + builder.set(k, values); + } else { + builder.set(k, v); + } + } + + public static StructuredRecord transform(Map object, Schema schema) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + object.entrySet().stream() + .filter(k -> schema.getField(k.getKey()) != null) // filter absent fields in the schema + .forEach(k -> transformValue(k.getKey(), k.getValue(), schema, builder)); + + return builder.build(); + } + + public static StructuredRecord transform(IBaseObject object, Schema schema) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + object.asFilteredMap(schema) + .forEach((k, v) -> transformValue(k, v, schema, builder)); + + return builder.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSplit.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSplit.java new file mode 100644 index 0000000..54db8fb --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSplit.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; + +/** + * A no-op split + */ +public class SendGridSplit extends InputSplit implements Writable { + public SendGridSplit() { + + } + + @Override + public void write(DataOutput dataOutput) { + + } + + @Override + public void readFields(DataInput dataInput) { + + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String[] getLocations() { + return new String[0]; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/APIResponseType.java b/src/main/java/io/cdap/plugin/sendgrid/common/APIResponseType.java new file mode 100644 index 0000000..2b853a7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/APIResponseType.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sendgrid.common; + +/** + * The way, how SendGrid API return the objects + * + */ +public enum APIResponseType { + /** + * Objects come to the response as part of the list + */ + LIST, + + /** + * Objects come as part of wrapper, which consists from the metadata and the result + */ + RESULT, + + /** + * Same as {@link APIResponseType#RESULT} + */ + RESULTS +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java new file mode 100644 index 0000000..a6c07ce --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java @@ -0,0 +1,331 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.sendgrid.Method; +import com.sendgrid.Request; +import com.sendgrid.Response; +import com.sendgrid.SendGrid; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import io.cdap.plugin.sendgrid.common.objects.BasicResult; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; +import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMail; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingNewContacts; + +import java.io.IOException; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.annotation.Nullable; + +/** + * SendGrid Client + */ +public class SendGridClient { + private static final String CONNECTION_CHECK_ENDPOINT = "alerts"; + + /** + * Extended version of the original SendGrid API wrapper with added support of basic auth + */ + private static class SendGridAPIClient extends SendGrid { + + SendGridAPIClient(String apiKey) { + super(apiKey); + } + + SendGridAPIClient(String username, String password) { + super(""); // actual key is not required due to it would be rewritten with basic auth data + initializeSendGrid(username, password); + } + + /** + * Replaces "Authorization" header, configured for the bearer auth with basic auth credentials + * + * @param username name of the user + * @param password password of the user + */ + private void initializeSendGrid(String username, String password) { + String encoding = Base64.getEncoder().encodeToString((String.format("%s:%s", username, password).getBytes())); + addRequestHeader("Authorization", String.format("Basic %s", encoding)); + } + } + + private SendGridAPIClient sendGrid; + private Gson gson; + + private SendGridClient() { + gson = new GsonBuilder().create(); + } + + public SendGridClient(BaseConfig config) { + this(); + if (config.getAuthType() == SendGridAuthType.API) { + sendGrid = new SendGridAPIClient(config.getSendGridApiKey()); + } else if (config.getAuthType() == SendGridAuthType.BASIC) { + sendGrid = new SendGridAPIClient(config.getAuthUserName(), config.getAuthPassword()); + } else { + throw new IllegalArgumentException(String.format("Invalid authentication method '%s'", + SendGridClient.class.getCanonicalName())); + } + } + + public SendGridClient(String key) { + this(); + sendGrid = new SendGridAPIClient(key); + } + + public SendGridClient(String username, String password) { + this(); + sendGrid = new SendGridAPIClient(username, password); + } + + /** + * Low level function to query API endpoints + * + * @param method the HTTP method to use + * @param endpoint relative uri to be queried + * @param arguments arguments for the query + * @param data + * @return query body + * @throws IOException if any issue with query the API happen + */ + private String makeApiRequest(Method method, String endpoint, @Nullable Map arguments, + String data) throws IOException { + + Request request = new Request(); + request.setMethod(method); + request.setEndpoint(endpoint); + + if ((method == Method.POST || method == Method.PUT) && !Strings.isNullOrEmpty(data)) { + request.setBody(data); + } + + if (arguments != null && !arguments.isEmpty()) { + arguments.forEach(request::addQueryParam); + } + + Response response; + try { + response = sendGrid.api(request); + } catch (IOException e) { + String serverMessage = String.format("Request to SendGrid API \"%s\"", endpoint); + // Parse error response from the server + if (e.getMessage().contains("Body:")) { + String[] messages = e.getMessage().split("Body:"); + HashMap>> errors = gson.fromJson( + messages[1], + new TypeToken>>>() { }.getType() + ); + if (errors.containsKey("errors")) { + String description = errors.get("errors").stream() + .filter(x -> x.containsKey("message")) + .map(x -> x.get("message")) + .collect(Collectors.joining(";")); + + serverMessage = String.format("%s, API response: %s", messages[0], description); + } + } + throw new IOException(serverMessage, e); + } + + return response.getBody(); + } + + /** + * Checks connection to the service by testing API endpoint, in case + * of exception would be generated {@link IOException} + */ + public void checkConnection() throws IOException { + makeApiRequest(Method.GET, CONNECTION_CHECK_ENDPOINT, null, null); + } + + /** + * Verify all incoming arguments for the query object + * + * @param objectInfo objects definition + * @param arguments query arguments + * @throws IllegalArgumentException if any validation issue + */ + private void checkIncomingArguments(ObjectInfo objectInfo, Map arguments) + throws IllegalArgumentException { + + if (objectInfo.getRequiredArguments() != null && !objectInfo.getRequiredArguments().isEmpty()) { + if (arguments == null || arguments.isEmpty()) { + throw new IllegalArgumentException(String.format( + "Object '%s' require input arguments to be passed, nothing found", + objectInfo.getCdapObjectName() + )); + } + List exceptions = new ArrayList<>(); + + objectInfo.getRequiredArguments().forEach(x -> { + try { + if (Strings.isNullOrEmpty(x)) { + return; + } + + arguments.keySet().stream() + .filter(x::equals) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(String.format( + "Object '%s' require %s argument, but nothing provided", + objectInfo.getCdapObjectName(), + x + ))); + } catch (IllegalArgumentException e) { + exceptions.add(e.getMessage()); + } + }); + if (!exceptions.isEmpty()) { + throw new IllegalArgumentException(exceptions.stream().collect(Collectors.joining(System.lineSeparator()))); + } + } + } + + /** + * Post a SendGrid mail + * + * @param mail Mail object to send + */ + public void sendMail(SendGridMail mail) throws IOException { + ObjectInfo objectInfo = ObjectHelper.getObjectInfo(SendGridMail.class); + String data = gson.toJson(mail); + + makeApiRequest(Method.POST, objectInfo.getSendGridAPIUrl(), null, data); + } + + /** + * Query SendGrid API using plugin meta objects + * + * @param objectInfo objects definition + * @param arguments query arguments + * @return object representation of the query + * @throws IOException if any issue with query the API happen + * @throws IllegalStateException unsupported response type caught + * @throws IllegalArgumentException if any validation issue + */ + public List getObject(ObjectInfo objectInfo, Map arguments) + throws IOException, IllegalStateException, IllegalArgumentException { + checkIncomingArguments(objectInfo, arguments); + + String endpoint = objectInfo.getSendGridAPIUrl(); + String response = makeApiRequest(Method.GET, endpoint, arguments, null); + Class clazz = objectInfo.getObjectClass(); + + if (objectInfo.getApiResponseType() == APIResponseType.RESULT) { + Type typeToken = new ParameterizedType() { + @Override + public Type[] getActualTypeArguments() { + return new Type[] { clazz }; + } + + @Override + public Type getRawType() { + return BasicResult.class; + } + + @Override + public Type getOwnerType() { + return null; + } + }; + BasicResult result = gson.fromJson(response, typeToken); + + return result.getResult(); + } else if (objectInfo.getApiResponseType() == APIResponseType.LIST) { + Type typeToken = new ParameterizedType() { + @Override + public Type[] getActualTypeArguments() { + return new Type[] { clazz }; + } + + @Override + public Type getRawType() { + return List.class; + } + + @Override + public Type getOwnerType() { + return null; + } + }; + + return gson.fromJson(response, typeToken); + } else { + throw new IllegalStateException(String.format( + "Unsupported API Response type: %s", + objectInfo.getApiResponseType().name() + )); + } + } + + /** + * Create contacts + * + * @param contacts list of contacts to be created + * @throws IOException if any issue with query the API happen + */ + public void createContacts(MarketingNewContacts contacts) throws IOException { + ObjectInfo objectInfo = ObjectHelper.getObjectInfoFromClass(MarketingNewContacts.class); + String data = gson.toJson(contacts); + makeApiRequest(Method.PUT, objectInfo.getSendGridAPIUrl(), null, data); + } + + /** + * Delete contacts by their id + * + * @param ids list of contact id + * @throws IOException if any issue with query the API happen + */ + public void deleteContacts(List ids) throws IOException { + ObjectInfo objectInfo = ObjectHelper.getObjectInfoFromClass(MarketingNewContacts.class); + String idsToRemove = String.join(",", ids); + Map args = new ImmutableMap.Builder() + .put("delete_all_contacts", "false") + .put("ids", idsToRemove) + .build(); + makeApiRequest(Method.DELETE, objectInfo.getSendGridAPIUrl(), args, null); + } + + /** + * Query SendGrid API using plugin meta objects + * + * @param objectInfo objects definition + * @return object representation of the query + * @throws IOException if any issue with query the API happen + * @throws IllegalStateException unsupported response type caught + * @throws IllegalArgumentException if any validation issue + */ + public List getObject(ObjectInfo objectInfo) + throws IOException, IllegalStateException, IllegalArgumentException { + + return getObject(objectInfo, null); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java new file mode 100644 index 0000000..5c111fb --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.config; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; + +import javax.annotation.Nullable; + +/** + * Provides all required configuration for reading SendGrid information + */ +public abstract class BaseConfig extends ReferencePluginConfig { + public static final String PLUGIN_NAME = "SendGrid"; + + public static final String PROPERTY_AUTH_TYPE = "authType"; + public static final String PROPERTY_SENDGRID_API_KEY = "sendGridApiKey"; + public static final String PROPERTY_AUTH_USERNAME = "username"; + public static final String PROPERTY_AUTH_PASSWORD = "password"; + + @Name(PROPERTY_AUTH_TYPE) + @Description("The way, how user would like to be authenticated to the SendGrid account") + @Macro + private String authType; + + @Name(PROPERTY_SENDGRID_API_KEY) + @Description("The SendGrid API Key taken from the SendGrid account") + @Macro + @Nullable + private String sendGridApiKey; + + @Name(PROPERTY_AUTH_USERNAME) + @Description("Login name for the SendGrid account") + @Macro + @Nullable + private String authUserName; + + @Name(PROPERTY_AUTH_PASSWORD) + @Description("Password for the SendGrid account") + @Macro + @Nullable + private String authPassword; + + /** + * Constructor + * + * @param referenceName uniquely identify source/sink for lineage, annotating metadata, etc. + */ + public BaseConfig(String referenceName) { + super(referenceName); + } + + /** + * Validate configuration for the issues + * + */ + protected abstract void validate(FailureCollector failureCollector); + + /** + * Client authentication way + */ + public SendGridAuthType getAuthType() { + switch (authType) { + case "api": + return SendGridAuthType.API; + case "basic": + return SendGridAuthType.BASIC; + default: + throw new IllegalArgumentException(String.format("Authentication using '%s' is not supported", authType)); + } + } + + /** + * Retrieves Api Key + */ + @Nullable + public String getSendGridApiKey() { + return sendGridApiKey; + } + + /** + * Retrieves username + */ + @Nullable + public String getAuthUserName() { + return authUserName; + } + + /** + * Retrieves password + */ + @Nullable + public String getAuthPassword() { + return authPassword; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfigValidator.java b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfigValidator.java new file mode 100644 index 0000000..43a4309 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfigValidator.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.config; + +import com.google.common.base.Strings; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.validation.ValidationFailure; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; + +import java.io.IOException; + +/** + * Validates configuration + */ +public abstract class BaseConfigValidator { + protected FailureCollector failureCollector; + protected SendGridClient client = null; + private BaseConfig config; + + public BaseConfigValidator(FailureCollector failureCollector, BaseConfig config) { + this.failureCollector = failureCollector; + this.config = config; + } + + private void checkAuthType() { + try { + config.getAuthType(); + } catch (IllegalArgumentException e) { + failureCollector.addFailure(String.format("Wrong authentication method selected: %s", e.getMessage()), null) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_TYPE); + } + } + + private void checkAuthData() { + boolean tryToLogin = true; + + switch (config.getAuthType()) { + case BASIC: + if (Strings.isNullOrEmpty(config.getAuthUserName())) { + failureCollector.addFailure("User name is not set", null) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_USERNAME); + tryToLogin = false; + } + if (Strings.isNullOrEmpty(config.getAuthPassword())) { + failureCollector.addFailure("Password is not set", null) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_PASSWORD); + tryToLogin = false; + } + + if (tryToLogin) { + client = new SendGridClient(config.getAuthUserName(), config.getAuthPassword()); + } + break; + case API: + if (Strings.isNullOrEmpty(config.getSendGridApiKey())) { + failureCollector.addFailure("API Key is not set", null) + .withConfigProperty(BaseConfig.PROPERTY_SENDGRID_API_KEY); + tryToLogin = false; + } + + if (tryToLogin) { + client = new SendGridClient(config.getSendGridApiKey()); + } + break; + } + } + + private void checkClientConnectivity() { + try { + client.checkConnection(); + } catch (IOException e) { + ValidationFailure failure = failureCollector + .addFailure(String.format("Connectivity issues: %s", e.getMessage()), null) + .withStacktrace(e.getStackTrace()); + + if (config.getAuthType() == SendGridAuthType.BASIC) { + failure + .withConfigProperty(BaseConfig.PROPERTY_AUTH_USERNAME) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_PASSWORD); + } + if (config.getAuthType() == SendGridAuthType.API) { + failure.withConfigProperty(BaseConfig.PROPERTY_SENDGRID_API_KEY); + } + } + } + + /** + * Perform validation tasks which did not involve API Client usage + */ + public abstract void doValidation(); + + public void validate() { + client = null; + + if (!config.containsMacro(BaseConfig.PROPERTY_AUTH_TYPE)) { + checkAuthType(); + } + if (!config.containsMacro(BaseConfig.PROPERTY_AUTH_USERNAME) && + !config.containsMacro(BaseConfig.PROPERTY_AUTH_PASSWORD) && + !config.containsMacro(BaseConfig.PROPERTY_SENDGRID_API_KEY)) { + checkAuthData(); + } + + doValidation(); + + // client could be not constructed, if any of checkAuth tests failed + if (client != null + && (!config.containsMacro(BaseConfig.PROPERTY_AUTH_USERNAME) && + !config.containsMacro(BaseConfig.PROPERTY_AUTH_PASSWORD) && + !config.containsMacro(BaseConfig.PROPERTY_SENDGRID_API_KEY) + )) { + checkClientConnectivity(); + } + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/BaseObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/BaseObject.java new file mode 100644 index 0000000..686404f --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/BaseObject.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.data.schema.Schema; + +import java.util.Map; +import java.util.Objects; + + +/** + * Base object for the {@link IBaseObject} interface + */ +public abstract class BaseObject implements IBaseObject { + + /** + * Return Map of fields according to provided schema + * + * @param schema object schema + * @return fields map + */ + @Override + public Map asFilteredMap(Schema schema) { + ImmutableMap.Builder fields = new ImmutableMap.Builder<>(); + Map allFields = asMap(); + + Objects.requireNonNull(schema.getFields()).stream() + .map(Schema.Field::getName) + .forEach(name -> fields.put(name, allFields.getOrDefault(name, new EmptyObject()))); + + return fields.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/EmptyObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/EmptyObject.java new file mode 100644 index 0000000..d31b2a3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/EmptyObject.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import java.util.Collections; +import java.util.Map; + +/** + * Provides empty stub object + */ +public class EmptyObject extends BaseObject implements IBaseObject { + /** + * Map of all object fields with values + */ + @Override + public Map asMap() { + return Collections.emptyMap(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/IBaseObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/IBaseObject.java new file mode 100644 index 0000000..4c5fa8a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/IBaseObject.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.cdap.api.data.schema.Schema; + +import java.util.Map; + +/** + * Interface for all SendGrid objects + * + * Any child, implemented current interface should provide all + * fields outside through {@link IBaseObject#asMap()} method. + * + * No getters allowed, unless any custom object usage planed. + */ +public interface IBaseObject { + /** + * Map of all object fields with values + */ + Map asMap(); + + /** + * Provide access to object fields mentioned in {@link Schema} + * + * @param schema generated or customized schema for the object + */ + Map asFilteredMap(Schema schema); +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/MultiObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/MultiObject.java new file mode 100644 index 0000000..47e59fd --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/MultiObject.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import java.util.HashMap; +import java.util.Map; + +/** + * Multi-Object Holder + */ +public class MultiObject extends BaseObject implements IBaseObject { + + private Map objects; + + public MultiObject() { + objects = new HashMap<>(); + } + + public void addObject(String name, IBaseObject object) { + objects.put(name, object); + } + + /** + * Map of all object fields with values + */ + @Override + public Map asMap() { + return objects; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectDefinition.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectDefinition.java new file mode 100644 index 0000000..e2aa416 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectDefinition.java @@ -0,0 +1,78 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Defines Entity Object with all related options which defines entity + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface ObjectDefinition { + /** + * Object definition type + */ + enum ObjectDefinitionType { + /** + * Describes top-level object + */ + BASE, + + /** + * Describes object, designed to be used as part of another objects + */ + NESTED, + + /** + * Custom standalone object + */ + CUSTOM + } + + /** + * Entity internal name + */ + String Name() default ""; + + /** + * One of the {@link DataSourceGroupType} + */ + DataSourceGroupType Group() default DataSourceGroupType.Other; + + /** + * Relative API URI for the Entity + */ + String APIUrl() default ""; + + /** + * The way how REST API provides the information + */ + APIResponseType APIResponseType() default APIResponseType.LIST; + + /** + * List of argument names, required to made request + */ + String[] RequiredArguments() default ""; + + ObjectDefinitionType ObjectType() default ObjectDefinitionType.BASE; +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldDefinition.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldDefinition.java new file mode 100644 index 0000000..d8fdf50 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldDefinition.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.cdap.api.data.schema.Schema; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Defines entity fields + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface ObjectFieldDefinition { + + Schema.Type FieldType() default Schema.Type.STRING; + + /** + * Internal name of the {@link ObjectDefinition#Name()} + *

+ * Only objects with type {@link io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition.ObjectDefinitionType#NESTED} + * allowed + */ + String NestedClass() default ""; +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldInfo.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldInfo.java new file mode 100644 index 0000000..68431a6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldInfo.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.cdap.api.data.schema.Schema; + +/** + * Entity meta-info holder for the {@link ObjectFieldDefinition} annotation + */ +public class ObjectFieldInfo { + private String name; + private Schema.Type type; + private String nestedClass; + + public ObjectFieldInfo(String name, Schema.Type type, String nestedClass) { + this.name = name; + this.type = type; + this.nestedClass = nestedClass; + } + + public String getName() { + return name; + } + + public Schema.Type getType() { + return type; + } + + public String getNestedClassName() { + return nestedClass; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java new file mode 100644 index 0000000..2b0d870 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java @@ -0,0 +1,273 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMail; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingAutomation; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingContacts; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSegments; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSenders; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSendersContact; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSendersVerified; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSingleSend; +import io.cdap.plugin.sendgrid.common.objects.stats.AdvancedStats; +import io.cdap.plugin.sendgrid.common.objects.stats.CategoryStats; +import io.cdap.plugin.sendgrid.common.objects.stats.GlobalStats; +import io.cdap.plugin.sendgrid.common.objects.stats.MetricStats; +import io.cdap.plugin.sendgrid.common.objects.stats.StatsStats; +import io.cdap.plugin.sendgrid.common.objects.suppressions.BounceSuppression; +import io.cdap.plugin.sendgrid.common.objects.suppressions.GlobalUnsubscribeSuppression; +import io.cdap.plugin.sendgrid.common.objects.suppressions.GroupUnsubscribeSuppression; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.Nullable; + +/** + * Schema processing core. Deals with such work as: + * - resolves schema from annotated entities + * - builds custom schemas with accounting base schema structure + * - support nesting entities, maps + * ToDo: add list support + */ +public class ObjectHelper { + + /* + Could be used Reflections to discover the objects, but it is additional + dependency and time to dynamically discover them + */ + private static List objects = Arrays.asList( + // base objects + MarketingAutomation.class, + MarketingContacts.class, + MarketingSegments.class, + MarketingSenders.class, + MarketingSingleSend.class, + AdvancedStats.class, + CategoryStats.class, + GlobalStats.class, + BounceSuppression.class, + GlobalUnsubscribeSuppression.class, + GroupUnsubscribeSuppression.class, + + // nested objects + MarketingSendersContact.class, + MarketingSendersVerified.class, + MetricStats.class, + StatsStats.class, + + // custom objects + SendGridMail.class + ); + + private static Map objectsDefinitions; + + static { + // resolves available entities schema on first access + buildSchemaDefinition(); + } + + public static ObjectInfo getObjectInfoFromClass(Class object) { + ObjectDefinition objectDefinition = (ObjectDefinition) object.getAnnotation(ObjectDefinition.class); + + List objectFieldInfos = Arrays.stream(object.getDeclaredFields()).map(x -> { + try { + String name = x.getAnnotation(SerializedName.class).value(); + ObjectFieldDefinition objectFieldDefinition = x.getAnnotation(ObjectFieldDefinition.class); + + return new ObjectFieldInfo(name, objectFieldDefinition.FieldType(), objectFieldDefinition.NestedClass()); + } catch (NullPointerException e) { + return null; // Ignore non-annotated fields + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + return new ObjectInfo( + objectDefinition.Name(), + objectFieldInfos, + objectDefinition.APIUrl(), + objectDefinition.APIResponseType(), + object, + objectDefinition.Group(), + Arrays.asList(objectDefinition.RequiredArguments()), + objectDefinition.ObjectType() + ); + } + + /** + * Create schema definition for annotated objects + */ + private static void buildSchemaDefinition() { + if (objectsDefinitions != null) { + return; + } + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + objects.forEach(object -> { + try { + builder.put(object.getName(), getObjectInfoFromClass(object)); + } catch (NullPointerException e) { + throw new RuntimeException(String.format("Object with name %s not annotated with %s", object.getName(), + ObjectDefinition.class.getName())); + } + }); + objectsDefinitions = builder.build(); + } + + public static List getObjectNames() { + return objectsDefinitions.values().stream() + .filter(x -> x.getObjectType() == ObjectDefinition.ObjectDefinitionType.BASE) + .map(ObjectInfo::getCdapObjectName) + .collect(Collectors.toList()); + } + + /** + * Provides entity schema definition + * + * @param objectClass entity class, which derived from {@link IBaseObject} + */ + public static ObjectInfo getObjectInfo(Class objectClass) { + return objectsDefinitions.get(objectClass.getName()); + } + + /** + * Provides entity schema definition + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + */ + @Nullable + public static ObjectInfo getObjectInfo(String internalObjectName) { + return objectsDefinitions.values().stream() + .filter(x -> !Strings.isNullOrEmpty(x.getCdapObjectName()) && x.getCdapObjectName().equals(internalObjectName)) + .findFirst() + .orElse(null); + } + + /** + * Provides schema definition for desired entities with only desired fields included + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + * @param requestedFields the names, provided via {@link SerializedName#value()} + * + * @return CDAP Schema + */ + public static Schema buildSchema(List internalObjectName, @Nullable List requestedFields) { + return buildSchema(internalObjectName, requestedFields, false); + } + + /** + * Provides schema definition for desired entities with only desired fields included + * + * For single requested object: + * Schema generator creates plain schema, where each field represents as column + * and whole schema describes only one entity + * + * For multi-object request: + * Schema generator creates additional top-level holder, where each column represents + * separate requested entity + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + * @param requestedFields the names, provided via {@link SerializedName#value()} + * @param alwaysMultiObject regulates how to generate schema if only one entity is requested + * + * @return CDAP Schema + */ + public static Schema buildSchema(List internalObjectName, @Nullable List requestedFields, + boolean alwaysMultiObject) { + // generate simple schema for the single object + if (!alwaysMultiObject && internalObjectName.size() == 1) { + return buildSchema(internalObjectName.get(0), requestedFields); + } + + List fields = new ArrayList<>(); + + internalObjectName.forEach(object -> { + List objectFields = buildSchema(object, requestedFields).getFields(); + Schema.Field field = Schema.Field.of(object, Schema.recordOf(object, Objects.requireNonNull(objectFields))); + + fields.add(field); + }); + + return Schema.recordOf("output", fields); + } + + /** + * Provides schema definition for desired entity with only desired fields included + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + * @param requestedFields the names, provided via {@link SerializedName#value()} + * + * @return CDAP Schema + */ + public static Schema buildSchema(String internalObjectName, @Nullable List requestedFields) { + ObjectInfo objectInfo = getObjectInfo(internalObjectName); + + if (objectInfo == null) { + return Schema.recordOf(internalObjectName); + } + + List fieldInfos; + + if (requestedFields == null || requestedFields.isEmpty()) { + fieldInfos = objectInfo.getFieldDefinitions(); + } else { + fieldInfos = objectInfo.getFieldsDefinitions(requestedFields); + if (fieldInfos.isEmpty()) { // if user selected no fields belonging to current object, show all fields + fieldInfos = objectInfo.getFieldDefinitions(); + } + } + + List cdapFields = fieldInfos.stream() + .map(x -> { + if (x.getType() == Schema.Type.MAP) { + if (Strings.isNullOrEmpty(x.getNestedClassName())) { + throw new IllegalArgumentException(String.format("Nested class is not declared for the field %s", + x.getName())); + } + List nestedFields = buildSchema(x.getNestedClassName(), requestedFields).getFields(); + + return Schema.Field.of(x.getName(), + Schema.recordOf(x.getName(), Objects.requireNonNull(nestedFields)) + ); + } + if (x.getType() == Schema.Type.ARRAY) { + if (Strings.isNullOrEmpty(x.getNestedClassName())) { + throw new IllegalArgumentException(String.format("Nested class is not declared for the field %s", + x.getName())); + } + List nestedFields = buildSchema(x.getNestedClassName(), requestedFields).getFields(); + return Schema.Field.of(x.getName(), + Schema.arrayOf( + Schema.recordOf(x.getName(), Objects.requireNonNull(nestedFields)) + ) + ); + } + return Schema.Field.of(x.getName(), Schema.nullableOf(Schema.of(x.getType()))); + }) + .collect(Collectors.toList()); + + return (cdapFields.isEmpty()) + ? Schema.recordOf(objectInfo.getCdapObjectName()) + : Schema.recordOf(objectInfo.getCdapObjectName(), cdapFields); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectInfo.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectInfo.java new file mode 100644 index 0000000..d6b6047 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectInfo.java @@ -0,0 +1,88 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Entity meta-info holder for {@link ObjectDefinition} + */ +public class ObjectInfo { + private String cdapObjectName; + + private String sendGridAPIUrl; + private DataSourceGroupType dataSourceGroupType; + private Class objectClass; + private List fieldDefinitions; + private APIResponseType apiResponseType; + private List requiredArguments; + private ObjectDefinition.ObjectDefinitionType objectType; + + public ObjectInfo(String cdapObjectName, List fieldDefinitions, String sendGridAPIUrl, + APIResponseType apiResponseType, Class objectClass, DataSourceGroupType dataSourceGroupType, + List requiredArguments, ObjectDefinition.ObjectDefinitionType objectType) { + this.cdapObjectName = cdapObjectName; + this.fieldDefinitions = fieldDefinitions; + this.sendGridAPIUrl = sendGridAPIUrl; + this.apiResponseType = apiResponseType; + this.objectClass = objectClass; + this.dataSourceGroupType = dataSourceGroupType; + this.requiredArguments = requiredArguments; + this.objectType = objectType; + } + + public String getCdapObjectName() { + return cdapObjectName; + } + + public String getSendGridAPIUrl() { + return sendGridAPIUrl; + } + + public List getFieldDefinitions() { + return fieldDefinitions; + } + + public List getFieldsDefinitions(List fields) { + return fieldDefinitions.stream() + .filter(x -> fields.stream().anyMatch(y -> x.getName().equals(y))) + .collect(Collectors.toList()); + } + + public Class getObjectClass() { + return objectClass; + } + + public DataSourceGroupType getDataSourceGroupType() { + return dataSourceGroupType; + } + + public APIResponseType getApiResponseType() { + return apiResponseType; + } + + public List getRequiredArguments() { + return requiredArguments.stream().filter(x -> !x.equals("")).collect(Collectors.toList()); + } + + public ObjectDefinition.ObjectDefinitionType getObjectType() { + return objectType; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicMetadata.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicMetadata.java new file mode 100644 index 0000000..c860a0b --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicMetadata.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects; + +import com.google.gson.annotations.SerializedName; + +/** + * SendGrid API Response Wrapper + */ +public class BasicMetadata { + + @SerializedName("self") + private String objectUrl; + + public String getSelfUrl() { + return objectUrl; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicResult.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicResult.java new file mode 100644 index 0000000..c3f8b32 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicResult.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sendgrid.common.objects; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * SendGrid API Response Wrapper + * + * @param Any {@link io.cdap.plugin.sendgrid.common.helpers.BaseObject} object + */ +public class BasicResult { + @SerializedName(value = "result") + private List result; + + @SerializedName(value = "results") + private List results; + + @Nullable + @SerializedName("_metadata") + private BasicMetadata metadata; + + public List getResult() { + return (results == null) ? result : results; + } + + @Nullable + public BasicMetadata getMetadata() { + return metadata; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java new file mode 100644 index 0000000..2e48e85 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects; + +import java.util.Arrays; + +/** + * Entities groups + */ +public enum DataSourceGroupType { + Marketing("MarketingCampaign"), + Stats("Statistic"), + Suppressions("suppression"), + Other("other"); + + private String value; + + private static String className = DataSourceGroupType.class.getName(); + + DataSourceGroupType(String dataSourceType) { + this.value = dataSourceType; + } + + public static DataSourceGroupType fromString(String value) { + return Arrays.stream(DataSourceGroupType.values()) + .filter(group -> group.value.equals(value)) + .findFirst() + .orElseThrow(() -> new IllegalStateException(String.format("'%s' is invalid %s", value, className))); + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/SendGridAuthType.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/SendGridAuthType.java new file mode 100644 index 0000000..71c05cf --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/SendGridAuthType.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects; + +/** + * SendGrid authentication way + */ +public enum SendGridAuthType { + /** + * API KEY + */ + API, + + /** + * HTTP Basic auth + */ + BASIC +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMail.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMail.java new file mode 100644 index 0000000..fc02338 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMail.java @@ -0,0 +1,144 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; + +import java.util.List; +import java.util.Map; + +/** + * SendGrid Mail + */ +@ObjectDefinition( + APIUrl = "mail/send", + ObjectType = ObjectDefinition.ObjectDefinitionType.CUSTOM +) +public class SendGridMail extends BaseObject implements IBaseObject { + + @SerializedName("personalizations") + private List personalizations; + + @SerializedName("from") + private SendGridMailPerson from; + + @SerializedName("reply_to") + private SendGridMailPerson replyTo; + + @SerializedName("subject") + private String subject; + + @SerializedName("content") + private List content; + + @SerializedName("mail_settings") + private SendGridMailSettings mailSettings; + + @SerializedName("tracking_settings") + private SendGridTrackingSettings trackingSettings; + + public SendGridMail(List personalizations, SendGridMailPerson from, + SendGridMailPerson replyTo, String subject, List content, + SendGridMailSettings mailSettings, SendGridTrackingSettings trackingSettings) { + this.personalizations = personalizations; + this.from = from; + this.replyTo = replyTo; + this.subject = subject; + this.content = content; + this.mailSettings = mailSettings; + this.trackingSettings = trackingSettings; + } + + SendGridMail() { + // no-op, required for the SendGridMailBuilder + } + + public List getPersonalizations() { + return personalizations; + } + + public SendGridMailPerson getFrom() { + return from; + } + + public SendGridMailPerson getReplyTo() { + return replyTo; + } + + public String getSubject() { + return subject; + } + + public List getContent() { + return content; + } + + public SendGridMailSettings getMailSettings() { + return mailSettings; + } + + public SendGridTrackingSettings getTrackingSettings() { + return trackingSettings; + } + + public void setPersonalizations(List personalizations) { + this.personalizations = personalizations; + } + + void setFrom(SendGridMailPerson from) { + this.from = from; + } + + void setReplyTo(SendGridMailPerson replyTo) { + this.replyTo = replyTo; + } + + void setSubject(String subject) { + this.subject = subject; + } + + void setContent(List content) { + this.content = content; + } + + void setMailSettings(SendGridMailSettings mailSettings) { + this.mailSettings = mailSettings; + } + + void setTrackingSettings(SendGridTrackingSettings trackingSettings) { + this.trackingSettings = trackingSettings; + } + + /** + * Map of all object fields with values + */ + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("personalization", personalizations) + .put("from", from) + .put("reply_to", replyTo) + .put("subject", subject) + .put("content", content) + .put("mail_settings", mailSettings) + .put("tracking_settings", trackingSettings) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailBuilder.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailBuilder.java new file mode 100644 index 0000000..64d5e94 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailBuilder.java @@ -0,0 +1,184 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.common.base.Strings; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * {@link SendGridMail} object builder + */ +public class SendGridMailBuilder { + private SendGridMail mail; + private SendGridPersonalizations personalizations; + + public static SendGridMailBuilder getInstance() { + return new SendGridMailBuilder(); + } + + private SendGridMailBuilder() { + this.mail = new SendGridMail(); + } + + private SendGridMailBuilder addContent(String type, @Nullable String content) { + if (Strings.isNullOrEmpty(content)) { + return this; + } + + List allContent = mail.getContent(); + if (mail.getContent() == null) { + allContent = new ArrayList<>(); + allContent.add(new SendGridMailContent(type, content)); + mail.setContent(allContent); + } else { + allContent.add(new SendGridMailContent(type, content)); + } + return this; + } + + private SendGridMailSettings getMailSetting() { + SendGridMailSettings settings = mail.getMailSettings(); + if (settings == null) { + settings = new SendGridMailSettings( + new SendGridMailFooter(false, null, null), + new SendGridSwitch(false) + ); + mail.setMailSettings(settings); + } + return settings; + } + + private SendGridPersonalizations getPersonalizations() { + if (personalizations == null) { + personalizations = new SendGridPersonalizations(); + mail.setPersonalizations(Collections.singletonList(personalizations)); + } + return personalizations; + } + + private SendGridTrackingSettings getTrackingSettings() { + SendGridTrackingSettings trackingSettings = mail.getTrackingSettings(); + if (trackingSettings == null) { + trackingSettings = new SendGridTrackingSettings( + new SendGridSwitch(false), + new SendGridSwitch(false), + new SendGridSwitch(false) + ); + mail.setTrackingSettings(trackingSettings); + } + return trackingSettings; + } + + public SendGridMailBuilder from(@Nullable String email) { + if (!Strings.isNullOrEmpty(email)) { + mail.setFrom(new SendGridMailPerson(email)); + } + return this; + } + + public SendGridMailBuilder subject(@Nullable String subject) { + if (!Strings.isNullOrEmpty(subject)) { + mail.setSubject(subject); + } + return this; + } + + public SendGridMailBuilder addTo(@Nullable String email) { + if (!Strings.isNullOrEmpty(email)) { + getPersonalizations().addTo(new SendGridMailPerson(email)); + } + return this; + } + public SendGridMailBuilder addBcc(@Nullable String email) { + if (!Strings.isNullOrEmpty(email)) { + getPersonalizations().addBcc(new SendGridMailPerson(email)); + } + return this; + } + + public SendGridMailBuilder addCc(@Nullable String email) { + if (!Strings.isNullOrEmpty(email)) { + getPersonalizations().addCc(new SendGridMailPerson(email)); + } + return this; + } + + public SendGridMailBuilder replyTo(@Nullable String email) { + if (!Strings.isNullOrEmpty(email)) { + mail.setReplyTo(new SendGridMailPerson(email)); + } + return this; + } + + public SendGridMailBuilder addTextContent(@Nullable String content) { + return addContent("text/plain", content); + } + + public SendGridMailBuilder addHtmlContent(@Nullable String content) { + return addContent("text/html", content); + } + + public SendGridMailBuilder sandboxMode(@Nullable Boolean enabled) { + if (enabled != null) { + getMailSetting().getSandboxMode().setEnable(enabled); + } + return this; + } + + public SendGridMailBuilder footerText(@Nullable Boolean enabled, @Nullable String content) { + if (enabled != null && !Strings.isNullOrEmpty(content)) { + getMailSetting().setFooter(new SendGridMailFooter(enabled, content, null)); + } + return this; + } + + public SendGridMailBuilder footerHtml(@Nullable Boolean enabled, @Nullable String content) { + if (enabled != null && !Strings.isNullOrEmpty(content)) { + getMailSetting().setFooter(new SendGridMailFooter(enabled, null, content)); + } + return this; + } + + public SendGridMailBuilder clickTracking(@Nullable Boolean enabled) { + if (enabled != null) { + getTrackingSettings().getClickTracking().setEnable(enabled); + } + return this; + } + + public SendGridMailBuilder openTracking(@Nullable Boolean enabled) { + if (enabled != null) { + getTrackingSettings().getOpenTracking().setEnable(enabled); + } + return this; + } + + public SendGridMailBuilder subscriptionTracking(@Nullable Boolean enabled) { + if (enabled != null) { + getTrackingSettings().getSubscriptionTracking().setEnable(enabled); + } + return this; + } + + public SendGridMail build() { + return mail; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailContent.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailContent.java new file mode 100644 index 0000000..71a2933 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailContent.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.gson.annotations.SerializedName; + +/** + * Mail Content + */ +public class SendGridMailContent { + @SerializedName("type") + private String type; + + @SerializedName("value") + private String value; + + public SendGridMailContent(String type, String value) { + this.type = type; + this.value = value; + } + + public String getType() { + return type; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailFooter.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailFooter.java new file mode 100644 index 0000000..37f9bea --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailFooter.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.gson.annotations.SerializedName; + +/** + * Mail footer + */ +public class SendGridMailFooter { + @SerializedName("enable") + private Boolean enable; + + @SerializedName("text") + private String text; + + @SerializedName("html") + private String html; + + public SendGridMailFooter(Boolean enable, String text, String html) { + this.enable = enable; + this.text = text; + this.html = html; + } + + public Boolean getEnable() { + return enable; + } + + public String getText() { + return text; + } + + public String getHtml() { + return html; + } + + public void setEnable(Boolean enable) { + this.enable = enable; + } + + public void setText(String text) { + this.text = text; + } + + public void setHtml(String html) { + this.html = html; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailPerson.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailPerson.java new file mode 100644 index 0000000..b7d61a4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailPerson.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.gson.annotations.SerializedName; + +/** + * Person object + */ +public class SendGridMailPerson { + @SerializedName("email") + private String email; + + public SendGridMailPerson(String email) { + this.email = email; + } + + public String getEmail() { + return email; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailSettings.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailSettings.java new file mode 100644 index 0000000..a31ab33 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailSettings.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.gson.annotations.SerializedName; + +/** + * Mail Settings + */ +public class SendGridMailSettings { + @SerializedName("footer") + private SendGridMailFooter footer; + + @SerializedName("sandbox_mode") + private SendGridSwitch sandboxMode; + + public SendGridMailSettings(SendGridMailFooter footer, SendGridSwitch sandboxMode) { + this.footer = footer; + this.sandboxMode = sandboxMode; + } + + public SendGridMailFooter getFooter() { + return footer; + } + + public SendGridSwitch getSandboxMode() { + return sandboxMode; + } + + public void setFooter(SendGridMailFooter footer) { + this.footer = footer; + } + + public void setSandboxMode(SendGridSwitch sandboxMode) { + this.sandboxMode = sandboxMode; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridPersonalizations.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridPersonalizations.java new file mode 100644 index 0000000..3217de4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridPersonalizations.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; + +/** + * Personalization Object + */ +public class SendGridPersonalizations { + @SerializedName("to") + private List to; + + @SerializedName("cc") + private List cc; + + @SerializedName("bcc") + private List bcc; + + public void addTo(SendGridMailPerson value) { + if (to == null) { + to = new ArrayList<>(); + } + to.add(value); + } + + public void addCc(SendGridMailPerson value) { + if (cc == null) { + cc = new ArrayList<>(); + } + cc.add(value); + } + + public void addBcc(SendGridMailPerson value) { + if (bcc == null) { + bcc = new ArrayList<>(); + } + bcc.add(value); + } + + public List getTo() { + return to; + } + + public List getCc() { + return cc; + } + + public List getBcc() { + return bcc; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridSwitch.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridSwitch.java new file mode 100644 index 0000000..e2ad777 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridSwitch.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.gson.annotations.SerializedName; + +/** + * Switch object + */ +public class SendGridSwitch { + + @SerializedName("enable") + private Boolean enable; + + public SendGridSwitch(Boolean enable) { + this.enable = enable; + } + + public Boolean getEnable() { + return enable; + } + + public void setEnable(Boolean enable) { + this.enable = enable; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridTrackingSettings.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridTrackingSettings.java new file mode 100644 index 0000000..62bd03a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridTrackingSettings.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.mail; + +import com.google.gson.annotations.SerializedName; + +/** + * Tracking settings + */ +public class SendGridTrackingSettings { + @SerializedName("click_tracking") + private SendGridSwitch clickTracking; + + @SerializedName("open_tracking") + private SendGridSwitch openTracking; + + @SerializedName("subscription_tracking") + private SendGridSwitch subscriptionTracking; + + public SendGridTrackingSettings(SendGridSwitch clickTracking, SendGridSwitch openTracking, + SendGridSwitch subscriptionTracking) { + this.clickTracking = clickTracking; + this.openTracking = openTracking; + this.subscriptionTracking = subscriptionTracking; + } + + public SendGridSwitch getClickTracking() { + return clickTracking; + } + + public SendGridSwitch getOpenTracking() { + return openTracking; + } + + public SendGridSwitch getSubscriptionTracking() { + return subscriptionTracking; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingAutomation.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingAutomation.java new file mode 100644 index 0000000..11d56a5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingAutomation.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * Automation Entity + */ +@ObjectDefinition( + Name = "Automation", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/automations" +) +public class MarketingAutomation extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("type") + private String type; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("status") + private String status; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @SerializedName("message_count") + private Integer messageCount; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("live_at") + private String liveAt; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("id", id) + .put("name", name) + .put("type", type) + .put("status", status) + .put("message_count", messageCount) + .put("created_at", createdAt) + .put("updated_at", updatedAt) + .put("live_at", liveAt) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingContacts.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingContacts.java new file mode 100644 index 0000000..8c661cc --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingContacts.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.BasicMetadata; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; + + +/** + * Contacts entity + */ +@ObjectDefinition( + Name = "Contacts", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/contacts", + APIResponseType = APIResponseType.RESULT +) +public class MarketingContacts extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + private String email; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("first_name") + private String firstName; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("last_name") + private String lastName; + + @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY) + @SerializedName("list_ids") + private List ids; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @SerializedName("_metadata") + private BasicMetadata metadata; + + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("created_at", createdAt) + .put("email", email) + .put("first_name", firstName) + .put("id", id) + .put("last_name", lastName) + .put("list_ids", ids) + .put("updated_at", updatedAt) + .build(); + } + +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContact.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContact.java new file mode 100644 index 0000000..1c4df04 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContact.java @@ -0,0 +1,120 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.gson.annotations.SerializedName; + +/** + * Object for creating new SendGrid contacts + */ +public class MarketingNewContact { + @SerializedName("address_line_1") + private String addressLine1; + + @SerializedName("address_line_2") + private String addressLine2; + + @SerializedName("city") + private String city; + + @SerializedName("country") + private String country; + + @SerializedName("email") + private String email; + + @SerializedName("first_name") + private String firstName; + + @SerializedName("last_name") + private String lastName; + + @SerializedName("postal_code") + private String postalCode; + + @SerializedName("state_province_region") + private String stateProvinceRegion; + + /** + * Initializes class from csv line. + * + * Column format: + * email,first_name,last_name,address_line_1,address_line_2,city,state_province_region,postal_code,country + */ + public MarketingNewContact(String csvLine) { + String[] columns = csvLine.split(","); + if (columns.length != 9) { + throw new IllegalArgumentException(String.format("Invalid csv formatted line: '%s'", csvLine)); + } + this.email = columns[0]; + this.firstName = columns[1]; + this.lastName = columns[2]; + this.addressLine1 = columns[3]; + this.addressLine2 = columns[4]; + this.city = columns[5]; + this.stateProvinceRegion = columns[6]; + this.postalCode = columns[7]; + this.country = columns[8]; + } + + public static MarketingNewContact fromCSVLine(String csv) { + return new MarketingNewContact(csv); + } + + public String getAddressLine1() { + return addressLine1; + } + + public String getCity() { + return city; + } + + public String getCountry() { + return country; + } + + public String getEmail() { + return email; + } + + public String getFirstName() { + return firstName; + } + + public String getLastName() { + return lastName; + } + + public String getPostalCode() { + return postalCode; + } + + public String getStateProvinceRegion() { + return stateProvinceRegion; + } + + public String getAddressLine2() { + return addressLine2; + } + + public void setLastName(String value) { + this.lastName = value; + } + + public void setEmail(String value) { + this.email = value; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContacts.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContacts.java new file mode 100644 index 0000000..fedd3f1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContacts.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.gson.annotations.SerializedName; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Object for creating new SendGrid contacts + */ +@ObjectDefinition(APIUrl = "marketing/contacts") +public class MarketingNewContacts extends BaseObject implements IBaseObject { + @SerializedName("list_ids") + private List listIds; + + @SerializedName("contacts") + private List contacts; + + public MarketingNewContacts(List csvLines) { + listIds = new ArrayList<>(); + contacts = csvLines.stream().map(MarketingNewContact::fromCSVLine).collect(Collectors.toList()); + } + + public MarketingNewContacts(String csvLines) { + this(Arrays.asList(csvLines.split(System.lineSeparator()))); + } + + @Override + public Map asMap() { + return null; + } + + public List getContacts() { + return contacts; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSegments.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSegments.java new file mode 100644 index 0000000..36c9f64 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSegments.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Segments entity + */ +@ObjectDefinition( + Name = "Segments", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/segments", + APIResponseType = APIResponseType.RESULT +) +public class MarketingSegments extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @SerializedName("contracts_count") + private int contractsCount; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("parent_list_id") + @Nullable + private String parentListId; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("sample_updated_at") + private String sampleUpdatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("contacts_count", contractsCount) + .put("created_at", createdAt) + .put("id", id) + .put("name", name) + .put("parent_list_id", (parentListId == null) ? "" : parentListId) + .put("sample_updated_at", sampleUpdatedAt) + .put("updated_at", updatedAt) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSenders.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSenders.java new file mode 100644 index 0000000..e62685b --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSenders.java @@ -0,0 +1,114 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * Senders entity + */ +@ObjectDefinition( + Name = "Senders", + Group = DataSourceGroupType.Marketing, + APIUrl = "mc/senders" +) +public class MarketingSenders extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("address") + private String address; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("address_2") + private String address2; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("city") + private String city; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("country") + private String country; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MarketingSendersContact") + @SerializedName("from") + private MarketingSendersContact from; + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("id") + private long id; + + @ObjectFieldDefinition(FieldType = Schema.Type.BOOLEAN) + @SerializedName("locked") + private boolean locked; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("nickname") + private String nickname; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MarketingSendersContact") + @SerializedName("reply_to") + private MarketingSendersContact replyTo; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("state") + private String state; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MarketingSendersVerified") + @SerializedName("verified") + private MarketingSendersVerified verified; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("zip") + private String zip; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("address", address) + .put("address_2", address2) + .put("city", city) + .put("country", country) + .put("created_at", createdAt) + .put("from", from.asMap()) + .put("id", id) + .put("locked", locked) + .put("nickname", nickname) + .put("reply_to", replyTo.asMap()) + .put("state", state) + .put("updated_at", updatedAt) + .put("verified", verified) + .put("zip", zip) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersContact.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersContact.java new file mode 100644 index 0000000..6732ba3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersContact.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; + +import java.util.Map; + +/** + * MarketingSendersContact nested entity + */ +@ObjectDefinition( + Name = "MarketingSendersContact", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class MarketingSendersContact extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + private String email; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("email", email) + .put("name", name) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersVerified.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersVerified.java new file mode 100644 index 0000000..00fc0dd --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersVerified.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * MarketingSendersVerified nested entity + */ +@ObjectDefinition( + Name = "MarketingSendersVerified", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class MarketingSendersVerified extends BaseObject implements IBaseObject { + + @Nullable + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("reason") + private String reason; + + @SerializedName("status") + @ObjectFieldDefinition(FieldType = Schema.Type.BOOLEAN) + private boolean status; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("reason", (reason == null) ? "" : reason) + .put("status", status) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSingleSend.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSingleSend.java new file mode 100644 index 0000000..7970415 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSingleSend.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * SingleSends nested entity + */ +@ObjectDefinition( + Name = "SingleSends", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/singlesends", + APIResponseType = APIResponseType.RESULT +) +public class MarketingSingleSend extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("status") + private String status; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("is_abtest") + private Boolean isAbtest; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("id", id) + .put("name", name) + .put("status", status) + .put("updated_at", updatedAt) + .put("created_at", createdAt) + .put("is_abtest", isAbtest.toString()) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java new file mode 100644 index 0000000..e98c2e4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.batch.source.SendGridSourceConfig; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * AdvancedStats entity + */ +@ObjectDefinition( + Name = "AdvancedStats", + Group = DataSourceGroupType.Stats, + APIUrl = "geo/stats", + RequiredArguments = { + SendGridSourceConfig.PROPERTY_START_DATE + } +) +public class AdvancedStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("date") + private String date; + + @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY, NestedClass = "StatsStats") + @SerializedName("stats") + private List stats; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("date", date) + .put("stats", stats.stream().map(StatsStats::asMap).collect(Collectors.toList())) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java new file mode 100644 index 0000000..94b1c02 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java @@ -0,0 +1,61 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.batch.source.SendGridSourceConfig; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * CategoryStats entity + */ +@ObjectDefinition( + Name = "CategoryStats", + Group = DataSourceGroupType.Stats, + APIUrl = "categories/stats", + RequiredArguments = { + SendGridSourceConfig.PROPERTY_START_DATE, + SendGridSourceConfig.PROPERTY_STAT_CATEGORIES + } +) +public class CategoryStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("date") + private String date; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP) + @SerializedName("stats") + private List stats; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("date", date) + .put("stats", stats.stream().map(StatsStats::asMap).collect(Collectors.toList())) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java new file mode 100644 index 0000000..4ca790d --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.batch.source.SendGridSourceConfig; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * GlobalStats entity + */ +@ObjectDefinition( + Name = "GlobalStats", + Group = DataSourceGroupType.Stats, + APIUrl = "stats", + RequiredArguments = { + SendGridSourceConfig.PROPERTY_START_DATE + } +) +public class GlobalStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("date") + private String date; + + @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY) + @SerializedName("stats") + private List stats; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("date", date) + .put("stats", stats.stream().map(MetricStats::asMap).collect(Collectors.toList())) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java new file mode 100644 index 0000000..390d91a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java @@ -0,0 +1,134 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * MetricStats entity + */ +@ObjectDefinition( + Name = "MetricStats", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class MetricStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("blocks") + private Integer blocks; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("bounce_drops") + private Integer bounceDrops; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("bounces") + private Integer bounces; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("clicks") + private Integer clicks; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("deferred") + private Integer deferred; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("invalid_emails") + private Integer invalidEmails; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("opens") + private Integer opens; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("processed") + private Integer processed; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("requests") + private Integer requests; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("spam_report_drops") + private Integer spamReportDrops; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("spam_reports") + private Integer spamReports; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("unique_clicks") + private Integer uniqueClicks; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("unique_opens") + private Integer uniqueOpens; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("unsubscribe_drops") + private Integer unsubscribeDrops; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @Nullable + @SerializedName("unsubscribes") + private Integer unsubscribes; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("blocks", (blocks == null) ? 0 : blocks) + .put("bounce_drops", (bounceDrops == null) ? 0 : bounceDrops) + .put("bounces", (bounces == null) ? 0 : bounces) + .put("clicks", (clicks == null) ? 0 : clicks) + .put("deferred", (deferred == null) ? 0 : deferred) + .put("invalid_emails", (invalidEmails == null) ? 0 : invalidEmails) + .put("opens", (opens == null) ? 0 : opens) + .put("processed", (processed == null) ? 0 : processed) + .put("requests", (requests == null) ? 0 : requests) + .put("spam_report_drops", (spamReportDrops == null) ? 0 : spamReportDrops) + .put("spam_reports", (spamReports == null) ? 0 : spamReports) + .put("unique_clicks", (uniqueClicks == null) ? 0 : uniqueClicks) + .put("unique_opens", (uniqueOpens == null) ? 0 : uniqueOpens) + .put("unsubscribe_drops", (unsubscribeDrops == null) ? 0 : unsubscribeDrops) + .put("unsubscribes", (unsubscribes == null) ? 0 : unsubscribes) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java new file mode 100644 index 0000000..300fd79 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; + +import java.util.Map; + +/** + * StatsStats entity + */ +@ObjectDefinition( + Name = "StatsStats", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class StatsStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MetricStats") + @SerializedName("metrics") + private MetricStats metrics; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("type") + private String type; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("metrics", metrics.asMap()) + .put("name", name) + .put("type", type) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/BounceSuppression.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/BounceSuppression.java new file mode 100644 index 0000000..4127fd6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/BounceSuppression.java @@ -0,0 +1,64 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.suppressions; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * Bounces entity + */ +@ObjectDefinition( + Name = "Bounces", + Group = DataSourceGroupType.Suppressions, + APIUrl = "suppression/bounces" +) +public class BounceSuppression extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("created") + private long created; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + private String email; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("reason") + private String reason; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("status") + private String status; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("created", created) + .put("email", email) + .put("reason", reason) + .put("status", status) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GlobalUnsubscribeSuppression.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GlobalUnsubscribeSuppression.java new file mode 100644 index 0000000..efabe8c --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GlobalUnsubscribeSuppression.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.suppressions; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * GlobalUnsubscribes entity + */ +@ObjectDefinition( + Name = "GlobalUnsubscribes", + Group = DataSourceGroupType.Suppressions, + APIUrl = "suppression/unsubscribes" +) +public class GlobalUnsubscribeSuppression extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("created") + public long created; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + public String email; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("created", created) + .put("email", email) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GroupUnsubscribeSuppression.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GroupUnsubscribeSuppression.java new file mode 100644 index 0000000..64d6816 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GroupUnsubscribeSuppression.java @@ -0,0 +1,77 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.suppressions; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * GroupUnsubscribes entity + */ +@ObjectDefinition( + Name = "GroupUnsubscribes", + Group = DataSourceGroupType.Suppressions, + APIUrl = "asm/groups" +) +public class GroupUnsubscribeSuppression extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("description") + private String description; + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("id") + private long id; + + @ObjectFieldDefinition(FieldType = Schema.Type.BOOLEAN) + @SerializedName("is_default") + private boolean isDefault; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @Nullable + @SerializedName("last_email_sent_at") + private String lastEmailSentAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("unsubscribes") + private long unsubscribes; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("description", description) + .put("id", id) + .put("is_default", isDefault) + .put("last_email_sent_at", (lastEmailSentAt == null) ? "" : lastEmailSentAt) + .put("name", name) + .put("unsubscribes", unsubscribes) + .build(); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/BaseTest.java b/src/test/java/io/cdap/plugin/sendgrid/BaseTest.java new file mode 100644 index 0000000..156716d --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/BaseTest.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * Base test class for SendGrid plugin + */ +public class BaseTest { + + /** + * Read resource as {@link String} + * + * @param name resource name + * @return resource string representation + * @throws IOException in case if resource not found + */ + public static String getResource(String name) throws IOException { + ClassLoader classLoader = BaseTest.class.getClassLoader(); + + try (InputStream inputStream = classLoader.getResourceAsStream(name)) { + if (inputStream == null) { + throw new IOException(String.format("Error in reading file '%s'", name)); + } + try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream)) { + BufferedReader buffer = new BufferedReader(inputStreamReader); + return buffer.lines().collect(Collectors.joining(System.lineSeparator())); + } + } + } + + /** + * Returns random uuid without "-" + */ + public static String getRandomUUID() { + return UUID.randomUUID().toString().replace("-", ""); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigTest.java b/src/test/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigTest.java new file mode 100644 index 0000000..8f7d1aa --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigTest.java @@ -0,0 +1,104 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.BaseTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + + +public class SendGridSinkConfigTest extends BaseTest { + private static Gson gson = new GsonBuilder().create(); + SendGridSinkConfig config; + + @Before + public void setUp() throws Exception { + config = gson.fromJson(getResource("SendGridSinkConfigExample.json"), SendGridSinkConfig.class); + } + + @Test + public void testGetMailSubject() { + Assert.assertEquals("subject", config.getMailSubject()); + } + + @Test + public void testGetFrom() { + Assert.assertEquals("test@email.com", config.getFrom()); + } + + @Test + public void testGetRecipientAddressSource() { + Assert.assertEquals(SendGridSinkConfig.ToAddressSource.INPUT, config.getRecipientAddressSource()); + } + + @Test + public void testGetRecipientAddresses() { + Assert.assertEquals( + Arrays.asList("test1@email.com", "test2@email.com"), + config.getRecipientAddresses() + ); + } + + @Test + public void testGetRecipientColumnName() { + Assert.assertEquals("column1", config.getRecipientColumnName()); + } + + @Test + public void testGetBodyColumnName() { + Assert.assertEquals("column2", config.getBodyColumnName()); + } + + @Test + public void testGetReplyTo() { + Assert.assertEquals("reply@email.com", config.getReplyTo()); + } + + @Test + public void testGetFooterEnable() { + Assert.assertEquals(true, config.getFooterEnable()); + } + + @Test + public void testGetFooterHTML() { + Assert.assertEquals("footer", config.getFooterHTML()); + } + + @Test + public void testGetSandboxMode() { + Assert.assertEquals(true, config.getSandboxMode()); + } + + @Test + public void testGetClickTracking() { + Assert.assertEquals(true, config.getClickTracking()); + } + + @Test + public void testGetOpenTracking() { + Assert.assertEquals(true, config.getOpenTracking()); + } + + @Test + public void testGetSubscriptionTracking() { + Assert.assertEquals(true, config.getSubscriptionTracking()); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigTest.java b/src/test/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigTest.java new file mode 100644 index 0000000..dee4d58 --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigTest.java @@ -0,0 +1,77 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.BaseTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +public class SendGridSourceConfigTest extends BaseTest { + private static Gson gson = new GsonBuilder().create(); + SendGridSourceConfig config; + + @Before + public void setUp() throws Exception { + config = gson.fromJson(getResource("SendGridSourceConfigExample.json"), SendGridSourceConfig.class); + } + + @Test + public void getFields() { + Assert.assertEquals( + Arrays.asList("address", "city", "contracts_count"), + config.getFields() + ); + } + + @Test + public void getDataSource() { + Assert.assertEquals( + Arrays.asList("SingleSends", "Senders", "CategoryStats", "Bounces", "GlobalUnsubscribes"), + config.getDataSource() + ); + } + + @Test + public void isMultiObjectMode() { + Assert.assertTrue(config.isMultiObjectMode()); + } + + @Test + public void getRequestArguments() { + Assert.assertEquals( + new ImmutableMap.Builder() + .put(SendGridSourceConfig.PROPERTY_START_DATE, "2019-09-18") + .put(SendGridSourceConfig.PROPERTY_END_DATE, "2019-09-21") + .put(SendGridSourceConfig.PROPERTY_STAT_CATEGORIES, "spam") + .build(), + config.getRequestArguments() + ); + } + + @Test + public void getDataSourceTypes() { + Assert.assertEquals( + Arrays.asList("MarketingCampaign", "Statistic", "Suppression"), + config.getDataSourceTypes() + ); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/common/config/BaseConfigTest.java b/src/test/java/io/cdap/plugin/sendgrid/common/config/BaseConfigTest.java new file mode 100644 index 0000000..90c4833 --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/common/config/BaseConfigTest.java @@ -0,0 +1,111 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.config; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.InstanceCreator; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.BaseTest; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.lang.reflect.Type; + + +public class BaseConfigTest extends BaseTest { + /** + * Custom class creator for abstract {@link BaseConfig} + */ + private static class BaseConfigInstanceCreator implements InstanceCreator { + + private String ref; + + BaseConfigInstanceCreator(String ref) { + this.ref = ref; + } + + @Override + public BaseConfig createInstance(Type type) { + return new BaseConfig(ref) { + @Override + protected void validate(FailureCollector failureCollector) { + // no-op + throw new IllegalArgumentException("no op method"); + } + }; + } + } + + private BaseConfig basicAuthConfig; + private BaseConfig keyAuthConfig; + + private static Gson gson; + private static String refName = "ref" + getRandomUUID(); + + @BeforeClass + public static void classSetUp() { + GsonBuilder builder = new GsonBuilder(); + builder.registerTypeAdapter(BaseConfig.class, new BaseConfigInstanceCreator(refName)); + gson = builder.create(); + } + + @Before + public void setUp() throws Exception { + basicAuthConfig = gson.fromJson(getResource("BaseConfigExample.json"), BaseConfig.class); + keyAuthConfig = gson.fromJson(getResource("KeyConfigExample.json"), BaseConfig.class); + } + + @Test + public void testReferenceValue() { + Assert.assertEquals(refName, basicAuthConfig.referenceName); + Assert.assertEquals(refName, keyAuthConfig.referenceName); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateBasic() { + basicAuthConfig.validate(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateKey() { + keyAuthConfig.validate(null); + } + + @Test + public void testGetAuthType() { + Assert.assertEquals(SendGridAuthType.BASIC, basicAuthConfig.getAuthType()); + Assert.assertEquals(SendGridAuthType.API, keyAuthConfig.getAuthType()); + } + + @Test + public void testGetSendGridApiKey() { + Assert.assertEquals("some-api-key", keyAuthConfig.getSendGridApiKey()); + } + + @Test + public void testGetAuthUserName() { + Assert.assertEquals("test user", basicAuthConfig.getAuthUserName()); + } + + @Test + public void testGetAuthPassword() { + Assert.assertEquals("test pass", basicAuthConfig.getAuthPassword()); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/etl/SendGridSourceTest.java b/src/test/java/io/cdap/plugin/sendgrid/etl/SendGridSourceTest.java new file mode 100644 index 0000000..93c15b7 --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/etl/SendGridSourceTest.java @@ -0,0 +1,181 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.etl; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.datapipeline.DataPipelineApp; +import io.cdap.cdap.datapipeline.SmartWorkflow; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.mock.batch.MockSink; +import io.cdap.cdap.etl.mock.test.HydratorTestBase; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.ProgramRunStatus; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.DataSetManager; +import io.cdap.cdap.test.WorkflowManager; +import io.cdap.plugin.sendgrid.BaseTest; +import io.cdap.plugin.sendgrid.batch.source.SendGridSource; +import io.cdap.plugin.sendgrid.batch.source.SendGridSourceConfig; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingContacts; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingNewContacts; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class SendGridSourceTest extends HydratorTestBase { + private static final ArtifactSummary APP_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); + private static final String LAST_NAME_FIELD = "last_name"; + private static final String ID_FIELD = "id"; + private static final String BASIC_AUTH_TYPE = "basic"; + private static final String REPLACE_PATTERN = "%id%"; + + private static String authType; + private static String authUser; + private static String authPass; + private static String authToken; + + private static SendGridClient client; + private static MarketingNewContacts contacts; + private static String reference; + private static int createdContactsAmount; + + @BeforeClass + public static void setupTestClass() throws Exception { + reference = BaseTest.getRandomUUID(); + authType = System.getProperty("sg.auth.type"); + if (authType != null && authType.equals(BASIC_AUTH_TYPE)) { + authUser = System.getProperty("sg.auth.user"); + authPass = System.getProperty("sg.auth.pass"); + + if (authUser == null || authPass == null) { + throw new IllegalArgumentException("'sg.auth.user' and 'sg.auth.pass' system property must not be empty"); + } + client = new SendGridClient(authUser, authPass); + } else { + authToken = System.getProperty("sg.auth.token"); + if (authToken == null) { + throw new IllegalArgumentException("'sg.auth.token' system property must not be empty"); + } + client = new SendGridClient(authToken); + } + + + ArtifactId parentArtifact = NamespaceId.DEFAULT.artifact(APP_ARTIFACT.getName(), APP_ARTIFACT.getVersion()); + setupBatchArtifacts(parentArtifact, DataPipelineApp.class); + addPluginArtifact( + NamespaceId.DEFAULT.artifact("example-plugins", "1.0.0"), + parentArtifact, + SendGridSource.class + ); + contacts = new MarketingNewContacts(BaseTest.getResource("new_contacts.csv")); + createdContactsAmount = contacts.getContacts().size(); + + contacts.getContacts().forEach(contact -> { + contact.setLastName(reference); + String mail = contact.getEmail(); + contact.setEmail(mail.replace(REPLACE_PATTERN, reference)); + }); + + client.createContacts(contacts); + } + + private static Stream> getContacts() throws IOException { + return client.getObject(ObjectHelper.getObjectInfo(MarketingContacts.class), null) + .stream() + .map(baseObject -> baseObject.asMap()) + .filter(fieldMap -> fieldMap.containsKey(LAST_NAME_FIELD) && fieldMap.get(LAST_NAME_FIELD).equals(reference)); + } + + @AfterClass + public static void tearDownTestClass() throws IOException { + List idsToRemove = getContacts() + .filter(fieldMap -> fieldMap.containsKey(ID_FIELD)) + .map(fieldMap -> (String) fieldMap.get(ID_FIELD)) + .filter(id -> id != null) + .collect(Collectors.toList()); + client.deleteContacts(idsToRemove); + } + + @Test + public void testBatchSource() throws Exception { + ImmutableMap.Builder optionsBuilder = new ImmutableMap.Builder<>(); + + if (authType == BASIC_AUTH_TYPE) { + optionsBuilder + .put(SendGridSourceConfig.PROPERTY_AUTH_TYPE, BASIC_AUTH_TYPE) + .put(SendGridSourceConfig.PROPERTY_AUTH_USERNAME, authUser) + .put(SendGridSourceConfig.PROPERTY_AUTH_PASSWORD, authPass); + } else { + optionsBuilder + .put(SendGridSourceConfig.PROPERTY_AUTH_TYPE, "api") + .put(SendGridSourceConfig.PROPERTY_SENDGRID_API_KEY, authToken); + + } + + optionsBuilder + .put("referenceName", "ref") + .put(SendGridSourceConfig.PROPERTY_DATA_SOURCE_TYPES, "MarketingCampaign") + .put(SendGridSourceConfig.PROPERTY_DATA_SOURCE_MARKETING, "Contacts") + .put(SendGridSourceConfig.PROPERTY_DATA_SOURCE_FIELDS, "created_at,email,first_name,last_name,updated_at"); + + ETLStage source = new ETLStage("name", new ETLPlugin(BaseConfig.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, + optionsBuilder.build(), null)); + ETLStage sink = new ETLStage("sink", MockSink.getPlugin("outputSink")); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(source) + .addStage(sink) + .addConnection(source.getName(), sink.getName()) + .build(); + + ApplicationId pipelineId = NamespaceId.DEFAULT.app("HttpBatch_"); + ApplicationManager appManager = deployApplication(pipelineId, new AppRequest<>(APP_ARTIFACT, etlConfig)); + + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.startAndWaitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); + + DataSetManager outputManager = getDataset("outputSink"); + List outputRecords = MockSink.readOutput(outputManager); + + int contactCount = client.getObject(ObjectHelper.getObjectInfo(MarketingContacts.class), null).size(); + int retrievedContactsCount = (int) outputRecords.stream() + .filter(x -> x.get("last_name").equals(reference)) + .count(); + + Assert.assertEquals(contactCount, outputRecords.size()); + Assert.assertEquals(getContacts().count(), retrievedContactsCount); + } +} diff --git a/src/test/resources/BaseConfigExample.json b/src/test/resources/BaseConfigExample.json new file mode 100644 index 0000000..9fca177 --- /dev/null +++ b/src/test/resources/BaseConfigExample.json @@ -0,0 +1,6 @@ +{ + "authType": "basic", + "sendGridApiKey": "", + "authUserName": "test user", + "authPassword": "test pass" +} \ No newline at end of file diff --git a/src/test/resources/KeyConfigExample.json b/src/test/resources/KeyConfigExample.json new file mode 100644 index 0000000..c4f2b72 --- /dev/null +++ b/src/test/resources/KeyConfigExample.json @@ -0,0 +1,6 @@ +{ + "authType": "api", + "sendGridApiKey": "some-api-key", + "authUserName": "", + "authPassword": "" +} \ No newline at end of file diff --git a/src/test/resources/SendGridSinkConfigExample.json b/src/test/resources/SendGridSinkConfigExample.json new file mode 100644 index 0000000..f4a93a2 --- /dev/null +++ b/src/test/resources/SendGridSinkConfigExample.json @@ -0,0 +1,15 @@ +{ + "from": "test@email.com", + "replyTo": "reply@email.com", + "footerEnable": "true", + "footerHTML": "footer", + "sandboxMode": "true", + "clickTracking": "true", + "openTracking": "true", + "subscriptionTracking": "true", + "recipientAddressSource": "input", + "recipientConfigAddressList": "test1@email.com,test2@email.com", + "recipientColumnName": "column1", + "bodyColumnName": "column2", + "mailSubject": "subject" +} \ No newline at end of file diff --git a/src/test/resources/SendGridSourceConfigExample.json b/src/test/resources/SendGridSourceConfigExample.json new file mode 100644 index 0000000..57f62d2 --- /dev/null +++ b/src/test/resources/SendGridSourceConfigExample.json @@ -0,0 +1,10 @@ +{ + "dataSourceTypes": "MarketingCampaign,Statistic,Suppression", + "dataSourceMarketing": "SingleSends,Senders", + "dataSourceStats": "CategoryStats", + "dataSourceSuppressions": "Bounces,GlobalUnsubscribes", + "dataSourceFields": "address,city,contracts_count", + "startDate": "2019-09-18", + "endDate": "2019-09-21", + "statCategories": "spam" +} \ No newline at end of file diff --git a/src/test/resources/new_contacts.csv b/src/test/resources/new_contacts.csv new file mode 100644 index 0000000..c2f1f82 --- /dev/null +++ b/src/test/resources/new_contacts.csv @@ -0,0 +1,4 @@ +user1.%id%@example.com,user1,user1,123 Neverland Lane,Suite 42,Denver,CO,80202,USA +user2.%id%@example.com,user2,user2,123 Neverland Lane,Suite 42,Denver,CO,80202,USA +user3.%id%@example.com,user3,user3,123 Neverland Lane,Suite 42,Denver,CO,80202,USA +user4.%id%@example.com,user4,user4,123 Neverland Lane,Suite 42,Denver,CO,80202,USA diff --git a/suppressions.xml b/suppressions.xml index dc4f081..fd0021b 100644 --- a/suppressions.xml +++ b/suppressions.xml @@ -1,14 +1,11 @@ + "-//Checkstyle//DTD SuppressionFilter Configuration 1.1//EN" + "https://checkstyle.org/dtds/suppressions_1_1.dtd"> - - - - - - - - - + + + - - - + + diff --git a/widgets/SendGrid-batchsink.json b/widgets/SendGrid-batchsink.json new file mode 100644 index 0000000..47ed0cd --- /dev/null +++ b/widgets/SendGrid-batchsink.json @@ -0,0 +1,282 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "name": "referenceName", + "label": "Reference Name", + "widget-type": "textbox" + }, + { + "name": "authType", + "label": "Authentication type", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "basic", + "options": [ + { + "id": "basic", + "label": "Basic" + }, + { + "id": "api", + "label": "API Key" + } + ] + } + }, + { + "name": "sendGridApiKey", + "label": "API Key", + "widget-type": "securekey-text", + "widget-attributes": { + "placeholder": "SendGrid API Key" + } + }, + { + "name": "username", + "label": "Username", + "widget-type": "textbox" + }, + { + "name": "password", + "label": "Password", + "widget-type": "password" + }, + { + "name": "from", + "label": "From", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "some@email.com" + } + }, + { + "name": "recipientAddressSource", + "label": "Recipient address source", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "input", + "options": [ + { + "id": "input", + "label": "Input Record" + }, + { + "id": "config", + "label": "Configuration" + } + ] + } + }, + { + "name": "recipientColumnName", + "label": "Column name for recipients addresses", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "recipients column name" + } + }, + { + "name": "recipientConfigAddressList", + "label": "Recipient address list", + "widget-type": "csv", + "widget-attributes": { + "delimiter": ",", + "value-placeholder": "some@mail.com" + } + }, + { + "name": "mailSubject", + "label": "Email subject", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "email subject" + } + }, + { + "name": "bodyColumnName", + "label": "Email body column name", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "body column name" + } + }, + { + "name": "replyTo", + "label": "Reply To", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "some@email.com" + } + } + ] + }, + { + "label": "Options", + "properties": [ + { + "name": "footerEnabled", + "label": "Custom Mail Footer", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "footerHtml", + "label": "Footer HTML", + "widget-type": "textarea", + "widget-attributes": { + "rows": 5, + "placeholder": "footer message" + } + }, + { + "name": "sandboxMode", + "label": "Sandbox Mode", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "clickTracking", + "label": "Click Tracking", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "openTracking", + "label": "Open Tracking", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "subscriptionTracking", + "label": "Subscription Tracking", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "schema-types": [ + "string" + ], + "schema-default-type": "string" + } + } + ], + "filters": [ + { + "name": "AuthTypeBasicFilter", + "condition": { + "expression": "authType == 'basic'" + }, + "show": [ + { + "name": "username", + "type": "property" + }, + { + "name": "password", + "type": "property" + } + ] + }, + { + "name": "AuthTypeAPIKeyFilter", + "condition": { + "expression": "authType == 'api'" + }, + "show": [ + { + "name": "sendGridApiKey", + "type": "property" + } + ] + }, + { + "name": "recipientSourceFilter", + "condition": { + "expression": "recipientAddressSource == 'input'" + }, + "show": [ + { + "name": "recipientColumnName", + "type": "property" + } + ] + }, + { + "name": "recipientSourceFilterConfig", + "condition": { + "expression": "recipientAddressSource == 'config'" + }, + "show": [ + { + "name": "recipientConfigAddressList", + "type": "property" + } + ] + } + ] +} \ No newline at end of file diff --git a/widgets/SendGrid-batchsource.json b/widgets/SendGrid-batchsource.json new file mode 100644 index 0000000..37809ec --- /dev/null +++ b/widgets/SendGrid-batchsource.json @@ -0,0 +1,364 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "name": "referenceName", + "label": "Reference Name", + "widget-type": "textbox" + }, + { + "name": "authType", + "label": "Authentication type", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "basic", + "options": [ + { + "id": "basic", + "label": "Basic" + }, + { + "id": "api", + "label": "API Key" + } + ] + } + }, + { + "name": "sendGridApiKey", + "label": "API Key", + "widget-type": "securekey-text", + "widget-attributes": { + "placeholder": "SendGrid API Key" + } + }, + { + "name": "username", + "label": "Username", + "widget-type": "textbox" + }, + { + "name": "password", + "label": "Password", + "widget-type": "password" + }, + { + "name": "dataSourceTypes", + "label": "Data Source Types", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "MarketingCampaign", + "label": "Marketing Campaign Objects" + }, + { + "id": "Statistic", + "label": "Statistic Objects" + }, + { + "id": "suppression", + "label": "Suppression Objects" + } + ], + "delimiter": "," + } + }, + { + "name": "dataSourceMarketing", + "label": "Marketing Campaign Objects", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "Automation", + "label": "Automation" + }, + { + "id": "Contacts", + "label": "Contacts" + }, + { + "id": "Segments", + "label": "Segments" + }, + { + "id": "Senders", + "label": "Senders" + }, + { + "id": "SingleSends", + "label": "Single Sends" + } + ], + "delimiter": "," + } + }, + { + "name": "dataSourceStats", + "label": "Statistic Objects", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "AdvancedStats", + "label": "Advanced Stats" + }, + { + "id": "CategoryStats", + "label": "Category Stats" + }, + { + "id": "GlobalStats", + "label": "Global Stats" + } + ], + "delimiter": "," + } + }, + { + "name": "dataSourceSuppressions", + "label": "Suppression Objects", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "Bounces", + "label": "Bounces" + }, + { + "id": "GlobalUnsubscribes", + "label": "Global Unsubscribes" + }, + { + "id": "GroupUnsubscribes", + "label": "Group Unsubscribes" + } + ], + "delimiter": "," + } + }, + { + "name": "dataSourceFields", + "label": "Fields", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id" : "address", + "label": "address" + }, + { + "id" : "address_2", + "label": "address_2" + }, + { + "id" : "city", + "label": "city" + }, + { + "id" : "contracts_count", + "label": "contracts_count" + }, + { + "id" : "country", + "label": "country" + }, + { + "id" : "created", + "label": "created" + }, + { + "id" : "created_at", + "label": "created_at" + }, + { + "id" : "date", + "label": "date" + }, + { + "id" : "description", + "label": "description" + }, + { + "id" : "email", + "label": "email" + }, + { + "id" : "first_name", + "label": "first_name" + }, + { + "id" : "from", + "label": "from" + }, + { + "id" : "id", + "label": "id" + }, + { + "id" : "is_abtest", + "label": "is_abtest" + }, + { + "id" : "is_default", + "label": "is_default" + }, + { + "id" : "last_email_sent_at", + "label": "last_email_sent_at" + }, + { + "id" : "last_name", + "label": "last_name" + }, + { + "id" : "list_ids", + "label": "list_ids" + }, + { + "id" : "live_at", + "label": "live_at" + }, + { + "id" : "locked", + "label": "locked" + }, + { + "id" : "message_count", + "label": "message_count" + }, + { + "id": "metrics", + "label": "metrics" + }, + { + "id" : "name", + "label": "name" + }, + { + "id" : "nickname", + "label": "nickname" + }, + { + "id" : "parent_list_id", + "label": "parent_list_id" + }, + { + "id" : "reason", + "label": "reason" + }, + { + "id" : "reply_to", + "label": "reply_to" + }, + { + "id" : "sample_updated_at", + "label": "sample_updated_at" + }, + { + "id" : "state", + "label": "state" + }, + { + "id" : "stats", + "label": "stats" + }, + { + "id" : "status", + "label": "status" + }, + { + "id" : "type", + "label": "type" + }, + { + "id" : "unsubscribes", + "label": "unsubscribes" + }, + { + "id" : "updated_at", + "label": "updated_at" + }, + { + "id" : "verified", + "label": "verified" + }, + { + "id" : "zip", + "label": "zip" + } + ], + "delimiter": "," + } + } + ] + }, + { + "label": "Options", + "properties": [ + { + "name": "start_date", + "label": "Start Date", + "widget-type": "textbox", + "placeholder": "YYYY-MM-DD" + }, + { + "name": "end_date", + "label": "End Date", + "widget-type": "textbox", + "placeholder": "YYYY-MM-DD" + }, + { + "name": "statCategories", + "label": "Statistic Categories", + "widget-type": "textbox", + "placeholder": "spam,..." + } + ] + } + ], + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": { + } + } + ], + "filters": [ + { + "name": "AuthTypeBasicFilter", + "condition": { + "expression": "authType == 'basic'" + }, + "show": [ + { + "name": "username", + "type": "property" + }, + { + "name": "password", + "type": "property" + } + ] + }, + { + "name": "AuthTypeAPIKeyFilter", + "condition": { + "expression": "authType == 'api'" + }, + "show": [ + { + "name": "sendGridApiKey", + "type": "property" + } + ] + } + ] +} \ No newline at end of file