From a1900eb960d8f0d3e720bbc06ceff635f54cd848 Mon Sep 17 00:00:00 2001 From: Dmitry Grinenko Date: Wed, 23 Oct 2019 00:17:37 +0300 Subject: [PATCH 1/6] General huge stuff - pom.xml - object definitions - icons - widget - *.xml --- .gitignore | 4 + checkstyle.xml | 387 +++++++++++++++++ docs/SendGrid-batchsource.md | 52 +++ icons/SendGrid-batchsource.png | Bin 0 -> 442 bytes pom.xml | 294 +++++++++++++ .../marketing/MarketingAutomation.java | 84 ++++ .../objects/marketing/MarketingContacts.java | 89 ++++ .../objects/marketing/MarketingSegments.java | 84 ++++ .../objects/marketing/MarketingSenders.java | 114 +++++ .../marketing/MarketingSendersContact.java | 52 +++ .../marketing/MarketingSendersVerified.java | 55 +++ .../marketing/MarketingSingleSend.java | 76 ++++ .../common/objects/stats/AdvancedStats.java | 59 +++ .../common/objects/stats/CategoryStats.java | 60 +++ .../common/objects/stats/GlobalStats.java | 57 +++ .../common/objects/stats/MetricStats.java | 117 +++++ .../common/objects/stats/StatsStats.java | 52 +++ .../suppressions/BounceSuppression.java | 64 +++ .../GlobalUnsubscribeSuppression.java | 54 +++ .../GroupUnsubscribeSuppression.java | 77 ++++ suppressions.xml | 30 ++ widgets/SendGrid-batchsource.json | 399 ++++++++++++++++++ 22 files changed, 2260 insertions(+) create mode 100644 checkstyle.xml create mode 100644 docs/SendGrid-batchsource.md create mode 100644 icons/SendGrid-batchsource.png create mode 100644 pom.xml create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingAutomation.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingContacts.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSegments.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSenders.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersContact.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersVerified.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSingleSend.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/BounceSuppression.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GlobalUnsubscribeSuppression.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GroupUnsubscribeSuppression.java create mode 100644 suppressions.xml create mode 100644 widgets/SendGrid-batchsource.json diff --git a/.gitignore b/.gitignore index a1c2a23..500cf5a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* + +# project files +*.iml +.idea \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..c77494c --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,387 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/SendGrid-batchsource.md b/docs/SendGrid-batchsource.md new file mode 100644 index 0000000..9eed6e3 --- /dev/null +++ b/docs/SendGrid-batchsource.md @@ -0,0 +1,52 @@ +# SendGrid batch source + +Description +----------- +This plugin used to query SendGrid v3 API. + +Properties +---------- +### General + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Authentication type:** The way, how user would like to be authenticated to the SendGrid acoount + +**API Key:** The SendGrid API Key taken from the SendGrid account + +**Username:** Login name for SendGrid + +**Password:** Login password for the username specified above + +**Data Source Types:** List of data source groups + +Available: +- Marketing Campaigns Fields +- Stats Fields +- Suppressions Fields + +**Data Source:** One of the above sources picked from list + +Available: +- Marketing Campaigns Fields + - Automation + - Single Sends + - Senders + - Contacts + - Segments +- Stats + - Global Stats + - Category Stats + - Advanced Stats +- Suppressions + - Bounces + - Global Unsubscribes + - Group Unsubscribes +**Data Source Fields:** + +**Start Date:** The date in format YYYY-MM-DD, starting from which the data is requested + +**End Date:** The date in format YYYY-MM-DD, the end date for the requested data + + + diff --git a/icons/SendGrid-batchsource.png b/icons/SendGrid-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..a716ff759ca777d88b4efeece431ee43a9c16f4f GIT binary patch literal 442 zcmeAS@N?(olHy`uVBq!ia0vp^MnJ63!3HE3me2eGq!^2X+?^QKos)S9a~60+7BevDDS z4D<|*O_-yrfTn7Bx;TbpIKQ1{?ak~c;kvw2Y*z$_SXaNBLB#6l$e;deaxZf!dgN@G zsCxT`#|9zAn{FxsjvI_x=Pt-eQ(MyMa^_Y)r_avB^~%dm+7`U}ExSz8kwfva_quzQ zr(Jh-T9mJyovq5bJNDhPw`W*ZtZZA6-E0wkHErjL1ato{^I~T*@SL2c=x3l3-DYuc zLXM=7Qq6n)79&fCuUV^9^KP*fF4H=DX=l^j9Y35s1nPe-{A0Evdi!Bz{dmLm_k!ZH z=ENNlXiG3U!q}z&B^vKK{I+(yzCh0Ii_gQQ{FPUJRm|(ZV)yD%^7+l_f9Fkoe8c~D zXW!-zUuOOa$_#ttc=+5OdmhG=4sHbT`l!YJsG#pF4o}$aWqokU$(3^zTPCb@?vQ_~ X)Vp=hlf>=7pk?rM^>bP0l+XkKSV^-~ literal 0 HcmV?d00001 diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..d2157e8 --- /dev/null +++ b/pom.xml @@ -0,0 +1,294 @@ + + + + 4.0.0 + + io.cdap.plugin + sendgrid + 1.3.0-SNAPSHOT + jar + SendGrid plugin + + + UTF-8 + 6.1.0-SNAPSHOT + 2.2.0-SNAPSHOT + 2.3.0 + 2.2.0 + 2.2.4 + 1.2 + + + + + + commons-logging + commons-logging + ${common.logging.version} + compile + + + com.sendgrid + sendgrid-java + 4.4.1 + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + com.google.code.gson + gson + ${gson.version} + compile + + + io.cdap.cdap + cdap-etl-api + ${cdap.version} + provided + + + com.google.code.gson + gson + + + + + io.cdap.plugin + hydrator-common + ${cdap.plugin.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.avro + avro + + + org.apache.zookeeper + zookeeper + + + com.google.guava + guava + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + servlet-api + javax.servlet + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + jasper-compiler + tomcat + + + jasper-runtime + tomcat + + + jsp-api + javax.servlet.jsp + + + slf4j-api + org.slf4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + org.slf4j + slf4j-log4j12 + + + com.google.inject.extensions + guice-servlet + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + + + + + + + org.apache.felix + maven-bundle-plugin + 3.3.0 + true + + + <_exportcontents> + io.cdap.plugin.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + io.cdap + cdap-maven-plugin + 1.1.0 + + + system:cdap-data-pipeline[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT) + + + + + create-artifact-config + prepare-package + + create-plugin-json + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + process-test-classes + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + **/org/apache/cassandra/**,**/org/apache/hadoop/** + + + check + + + + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + + + \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingAutomation.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingAutomation.java new file mode 100644 index 0000000..11d56a5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingAutomation.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * Automation Entity + */ +@ObjectDefinition( + Name = "Automation", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/automations" +) +public class MarketingAutomation extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("type") + private String type; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("status") + private String status; + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @SerializedName("message_count") + private Integer messageCount; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("live_at") + private String liveAt; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("id", id) + .put("name", name) + .put("type", type) + .put("status", status) + .put("message_count", messageCount) + .put("created_at", createdAt) + .put("updated_at", updatedAt) + .put("live_at", liveAt) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingContacts.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingContacts.java new file mode 100644 index 0000000..8c661cc --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingContacts.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.BasicMetadata; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; + + +/** + * Contacts entity + */ +@ObjectDefinition( + Name = "Contacts", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/contacts", + APIResponseType = APIResponseType.RESULT +) +public class MarketingContacts extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + private String email; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("first_name") + private String firstName; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("last_name") + private String lastName; + + @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY) + @SerializedName("list_ids") + private List ids; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @SerializedName("_metadata") + private BasicMetadata metadata; + + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("created_at", createdAt) + .put("email", email) + .put("first_name", firstName) + .put("id", id) + .put("last_name", lastName) + .put("list_ids", ids) + .put("updated_at", updatedAt) + .build(); + } + +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSegments.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSegments.java new file mode 100644 index 0000000..36c9f64 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSegments.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Segments entity + */ +@ObjectDefinition( + Name = "Segments", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/segments", + APIResponseType = APIResponseType.RESULT +) +public class MarketingSegments extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.INT) + @SerializedName("contracts_count") + private int contractsCount; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("parent_list_id") + @Nullable + private String parentListId; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("sample_updated_at") + private String sampleUpdatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("contacts_count", contractsCount) + .put("created_at", createdAt) + .put("id", id) + .put("name", name) + .put("parent_list_id", (parentListId == null) ? "" : parentListId) + .put("sample_updated_at", sampleUpdatedAt) + .put("updated_at", updatedAt) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSenders.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSenders.java new file mode 100644 index 0000000..e62685b --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSenders.java @@ -0,0 +1,114 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * Senders entity + */ +@ObjectDefinition( + Name = "Senders", + Group = DataSourceGroupType.Marketing, + APIUrl = "mc/senders" +) +public class MarketingSenders extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("address") + private String address; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("address_2") + private String address2; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("city") + private String city; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("country") + private String country; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MarketingSendersContact") + @SerializedName("from") + private MarketingSendersContact from; + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("id") + private long id; + + @ObjectFieldDefinition(FieldType = Schema.Type.BOOLEAN) + @SerializedName("locked") + private boolean locked; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("nickname") + private String nickname; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MarketingSendersContact") + @SerializedName("reply_to") + private MarketingSendersContact replyTo; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("state") + private String state; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MarketingSendersVerified") + @SerializedName("verified") + private MarketingSendersVerified verified; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("zip") + private String zip; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("address", address) + .put("address_2", address2) + .put("city", city) + .put("country", country) + .put("created_at", createdAt) + .put("from", from.asMap()) + .put("id", id) + .put("locked", locked) + .put("nickname", nickname) + .put("reply_to", replyTo.asMap()) + .put("state", state) + .put("updated_at", updatedAt) + .put("verified", verified) + .put("zip", zip) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersContact.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersContact.java new file mode 100644 index 0000000..6732ba3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersContact.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; + +import java.util.Map; + +/** + * MarketingSendersContact nested entity + */ +@ObjectDefinition( + Name = "MarketingSendersContact", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class MarketingSendersContact extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + private String email; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("email", email) + .put("name", name) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersVerified.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersVerified.java new file mode 100644 index 0000000..00fc0dd --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSendersVerified.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * MarketingSendersVerified nested entity + */ +@ObjectDefinition( + Name = "MarketingSendersVerified", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class MarketingSendersVerified extends BaseObject implements IBaseObject { + + @Nullable + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("reason") + private String reason; + + @SerializedName("status") + @ObjectFieldDefinition(FieldType = Schema.Type.BOOLEAN) + private boolean status; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("reason", (reason == null) ? "" : reason) + .put("status", status) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSingleSend.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSingleSend.java new file mode 100644 index 0000000..7970415 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingSingleSend.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * SingleSends nested entity + */ +@ObjectDefinition( + Name = "SingleSends", + Group = DataSourceGroupType.Marketing, + APIUrl = "marketing/singlesends", + APIResponseType = APIResponseType.RESULT +) +public class MarketingSingleSend extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("created_at") + private String createdAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("id") + private String id; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("status") + private String status; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("updated_at") + private String updatedAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("is_abtest") + private Boolean isAbtest; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("id", id) + .put("name", name) + .put("status", status) + .put("updated_at", updatedAt) + .put("created_at", createdAt) + .put("is_abtest", isAbtest.toString()) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java new file mode 100644 index 0000000..a1dbcee --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * AdvancedStats entity + */ +@ObjectDefinition( + Name = "AdvancedStats", + Group = DataSourceGroupType.Stats, + APIUrl = "geo/stats", + RequiredArguments = { + "start_date" // argument format: YYYY-MM-DD + } +) +public class AdvancedStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("date") + private String date; + + @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY) + @SerializedName("stats") + private List stats; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("date", date) + .put("stats", stats.stream().map(StatsStats::asMap).collect(Collectors.toList())) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java new file mode 100644 index 0000000..73a21d5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * CategoryStats entity + */ +@ObjectDefinition( + Name = "CategoryStats", + Group = DataSourceGroupType.Stats, + APIUrl = "categories/stats", + RequiredArguments = { + "start_date", // argument format: YYYY-MM-DD + "categories" + } +) +public class CategoryStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("date") + private String date; + + @ObjectFieldDefinition(FieldType = Schema.Type.MAP) + @SerializedName("stats") + private List stats; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("date", date) + .put("stats", stats.stream().map(StatsStats::asMap).collect(Collectors.toList())) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java new file mode 100644 index 0000000..52a26e6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * GlobalStats entity + */ +@ObjectDefinition( + Name = "GlobalStats", + Group = DataSourceGroupType.Stats, + APIUrl = "stats", + RequiredArguments = {"start_date"} +) +public class GlobalStats extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("date") + private String date; + + @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY) + @SerializedName("stats") + private List stats; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("date", date) + .put("stats", stats.stream().map(MetricStats::asMap).collect(Collectors.toList())) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java new file mode 100644 index 0000000..caf7510 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java @@ -0,0 +1,117 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * MetricStats entity + */ +@ObjectDefinition( + Name = "MetricStats", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class MetricStats extends BaseObject implements IBaseObject { + + @Nullable + @SerializedName("blocks") + private Integer blocks; + + @Nullable + @SerializedName("bounce_drops") + private Integer bounceDrops; + + @Nullable + @SerializedName("bounces") + private Integer bounces; + + @Nullable + @SerializedName("clicks") + private Integer clicks; + + @Nullable + @SerializedName("deferred") + private Integer deferred; + + @Nullable + @SerializedName("invalid_emails") + private Integer invalidEmails; + + @Nullable + @SerializedName("opens") + private Integer opens; + + @Nullable + @SerializedName("processed") + private Integer processed; + + @Nullable + @SerializedName("requests") + private Integer requests; + + @Nullable + @SerializedName("spam_report_drops") + private Integer spamReportDrops; + + @Nullable + @SerializedName("spam_reports") + private Integer spamReports; + + @Nullable + @SerializedName("unique_clicks") + private Integer uniqueClicks; + + @Nullable + @SerializedName("unique_opens") + private Integer uniqueOpens; + + @Nullable + @SerializedName("unsubscribe_drops") + private Integer unsubscribeDrops; + + @Nullable + @SerializedName("unsubscribes") + private Integer unsubscribes; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("blocks", (blocks == null) ? 0 : blocks) + .put("bounce_drops", (bounceDrops == null) ? 0 : bounceDrops) + .put("bounces", (bounces == null) ? 0 : bounces) + .put("clicks", (clicks == null) ? 0 : clicks) + .put("deferred", (deferred == null) ? 0 : deferred) + .put("invalid_emails", (invalidEmails == null) ? 0 : invalidEmails) + .put("opens", (opens == null) ? 0 : opens) + .put("processed", (processed == null) ? 0 : processed) + .put("requests", (requests == null) ? 0 : requests) + .put("spam_report_drops", (spamReportDrops == null) ? 0 : spamReportDrops) + .put("spam_reports", (spamReports == null) ? 0 : spamReports) + .put("unique_clicks", (uniqueClicks == null) ? 0 : uniqueClicks) + .put("unique_opens", (uniqueOpens == null) ? 0 : uniqueOpens) + .put("unsubscribe_drops", (unsubscribeDrops == null) ? 0 : unsubscribeDrops) + .put("unsubscribes", (unsubscribes == null) ? 0 : unsubscribes) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java new file mode 100644 index 0000000..8c7ace4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.stats; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; + +import java.util.Map; + +/** + * StatsStats entity + */ +@ObjectDefinition( + Name = "StatsStats", + ObjectType = ObjectDefinition.ObjectDefinitionType.NESTED +) +public class StatsStats extends BaseObject implements IBaseObject { + + @SerializedName("metrics") + private MetricStats metrics; + + @SerializedName("name") + private String name; + + @SerializedName("type") + private String type; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("metrics", metrics.asMap()) + .put("name", name) + .put("type", type) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/BounceSuppression.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/BounceSuppression.java new file mode 100644 index 0000000..4127fd6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/BounceSuppression.java @@ -0,0 +1,64 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.suppressions; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * Bounces entity + */ +@ObjectDefinition( + Name = "Bounces", + Group = DataSourceGroupType.Suppressions, + APIUrl = "suppression/bounces" +) +public class BounceSuppression extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("created") + private long created; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + private String email; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("reason") + private String reason; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("status") + private String status; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("created", created) + .put("email", email) + .put("reason", reason) + .put("status", status) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GlobalUnsubscribeSuppression.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GlobalUnsubscribeSuppression.java new file mode 100644 index 0000000..efabe8c --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GlobalUnsubscribeSuppression.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.suppressions; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +/** + * GlobalUnsubscribes entity + */ +@ObjectDefinition( + Name = "GlobalUnsubscribes", + Group = DataSourceGroupType.Suppressions, + APIUrl = "suppression/unsubscribes" +) +public class GlobalUnsubscribeSuppression extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("created") + public long created; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("email") + public String email; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("created", created) + .put("email", email) + .build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GroupUnsubscribeSuppression.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GroupUnsubscribeSuppression.java new file mode 100644 index 0000000..64d6816 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/suppressions/GroupUnsubscribeSuppression.java @@ -0,0 +1,77 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.suppressions; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * GroupUnsubscribes entity + */ +@ObjectDefinition( + Name = "GroupUnsubscribes", + Group = DataSourceGroupType.Suppressions, + APIUrl = "asm/groups" +) +public class GroupUnsubscribeSuppression extends BaseObject implements IBaseObject { + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("description") + private String description; + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("id") + private long id; + + @ObjectFieldDefinition(FieldType = Schema.Type.BOOLEAN) + @SerializedName("is_default") + private boolean isDefault; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @Nullable + @SerializedName("last_email_sent_at") + private String lastEmailSentAt; + + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) + @SerializedName("name") + private String name; + + @ObjectFieldDefinition(FieldType = Schema.Type.LONG) + @SerializedName("unsubscribes") + private long unsubscribes; + + @Override + public Map asMap() { + return new ImmutableMap.Builder() + .put("description", description) + .put("id", id) + .put("is_default", isDefault) + .put("last_email_sent_at", (lastEmailSentAt == null) ? "" : lastEmailSentAt) + .put("name", name) + .put("unsubscribes", unsubscribes) + .build(); + } +} diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..4641fb1 --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/widgets/SendGrid-batchsource.json b/widgets/SendGrid-batchsource.json new file mode 100644 index 0000000..4042bf6 --- /dev/null +++ b/widgets/SendGrid-batchsource.json @@ -0,0 +1,399 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "configuration-groups": [ + { + "label": "General", + "properties": [ + { + "name": "referenceName", + "label": "Reference Name", + "widget-type": "textbox" + }, + { + "name": "authType", + "label": "Authentication type", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "basic", + "options": [ + { + "id": "basic", + "label": "Basic" + }, + { + "id": "api", + "label": "API Key" + } + ] + } + }, + { + "name": "sendGridApiKey", + "label": "API Key", + "widget-type": "textbox", + "default": "" + }, + { + "name": "username", + "label": "Username", + "widget-type": "textbox", + "default": "" + }, + { + "name": "password", + "label": "Password", + "widget-type": "password", + "default": "" + }, + { + "name": "dataSourceTypes", + "label": "Data Source Types", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "campaigns", + "label": "Marketing Campaign Objects" + }, + { + "id": "stats", + "label": "Statistic Objects" + }, + { + "id": "suppressions", + "label": "Suppression Objects" + } + ], + "delimiter": "," + } + }, + { + "name": "dataSourceMarketing", + "label": "Marketing Campaign Objects", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "Automation", + "label": "Automation" + }, + { + "id": "Contacts", + "label": "Contacts" + }, + { + "id": "Segments", + "label": "Segments" + }, + { + "id": "Senders", + "label": "Senders" + }, + { + "id": "SingleSends", + "label": "Single Sends" + } + ], + "delimiter": ",", + "default": "" + } + }, + { + "name": "dataSourceStats", + "label": "Statistic Objects", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "AdvancedStats", + "label": "Advanced Stats" + }, + { + "id": "CategoryStats", + "label": "Category Stats" + }, + { + "id": "GlobalStats", + "label": "Advanced Stats" + }, + { + "id": "_suppressions", + "label": "Suppression Objects" + }, + { + "id": "Bounces", + "label": "Bounces" + }, + { + "id": "GlobalUnsubscribes", + "label": "Global Unsubscribes" + }, + { + "id": "GroupUnsubscribes", + "label": "Group Unsubscribes" + } + ], + "delimiter": ",", + "default": "" + } + }, + { + "name": "dataSourceSuppressions", + "label": "Suppression Objects", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id": "Bounces", + "label": "Bounces" + }, + { + "id": "GlobalUnsubscribes", + "label": "Global Unsubscribes" + }, + { + "id": "GroupUnsubscribes", + "label": "Group Unsubscribes" + } + ], + "delimiter": ",", + "default": "" + } + }, + { + "name": "dataSourceFields", + "label": "Fields", + "widget-type": "multi-select", + "widget-attributes": { + "options": [ + { + "id" : "address", + "label": "address" + }, + { + "id" : "address_2", + "label": "address_2" + }, + { + "id" : "city", + "label": "city" + }, + { + "id" : "contracts_count", + "label": "contracts_count" + }, + { + "id" : "country", + "label": "country" + }, + { + "id" : "created", + "label": "created" + }, + { + "id" : "created_at", + "label": "created_at" + }, + { + "id" : "date", + "label": "date" + }, + { + "id" : "description", + "label": "description" + }, + { + "id" : "email", + "label": "email" + }, + { + "id" : "first_name", + "label": "first_name" + }, + { + "id" : "from", + "label": "from" + }, + { + "id" : "id", + "label": "id" + }, + { + "id" : "is_abtest", + "label": "is_abtest" + }, + { + "id" : "is_default", + "label": "is_default" + }, + { + "id" : "last_email_sent_at", + "label": "last_email_sent_at" + }, + { + "id" : "last_name", + "label": "last_name" + }, + { + "id" : "list_ids", + "label": "list_ids" + }, + { + "id" : "live_at", + "label": "live_at" + }, + { + "id" : "locked", + "label": "locked" + }, + { + "id" : "message_count", + "label": "message_count" + }, + { + "id" : "name", + "label": "name" + }, + { + "id" : "nickname", + "label": "nickname" + }, + { + "id" : "parent_list_id", + "label": "parent_list_id" + }, + { + "id" : "reason", + "label": "reason" + }, + { + "id" : "reply_to", + "label": "reply_to" + }, + { + "id" : "sample_updated_at", + "label": "sample_updated_at" + }, + { + "id" : "state", + "label": "state" + }, + { + "id" : "stats", + "label": "stats" + }, + { + "id" : "status", + "label": "status" + }, + { + "id" : "type", + "label": "type" + }, + { + "id" : "unsubscribes", + "label": "unsubscribes" + }, + { + "id" : "updated_at", + "label": "updated_at" + }, + { + "id" : "verified", + "label": "verified" + }, + { + "id" : "zip", + "label": "zip" + } + ], + "delimiter": ",", + "defualt": "" + } + }, + { + "name": "startDate", + "label": "Start Date", + "widget-type": "textbox", + "default": "" + }, + { + "name": "endDate", + "label": "End Date", + "widget-type": "textbox", + "defaut": "" + } + ] + } + ], + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": { + } + } + ], + "filters": [ + { + "name": "AuthTypeBasicFilter", + "condition": { + "expression": "authType == 'basic'" + }, + "show": [ + { + "name": "username", + "type": "property" + }, + { + "name": "password", + "type": "property" + } + ] + }, + { + "name": "AuthTypeAPIKeyFilter", + "condition": "authType == 'api'", + "show": [ + { + "name": "sendGridApiKey", + "type": "property" + } + ] + }, + { + "name": "MarketingFilter", + "condition": "dataSourceTypes.includes('campaigns')", + "show": [ + { + "name": "dataSourceMarketing", + "type": "property" + } + ] + }, + { + "name": "StatsFilter", + "condition": "dataSourceTypes.includes('stats')", + "show": [ + { + "name": "dataSourceStats", + "type": "property" + } + ] + }, + { + "name": "SuppressionsFilter", + "condition": "dataSourceTypes.includes('suppressions')", + "show": [ + { + "name": "dataSourceSuppressions", + "type": "property" + } + ] + } + + ] + +} \ No newline at end of file From 4250a33ad9d557eb0c524752a52db6fd40cc6b1c Mon Sep 17 00:00:00 2001 From: Dmitry Grinenko Date: Wed, 23 Oct 2019 00:20:18 +0300 Subject: [PATCH 2/6] Core functionality --- docs/SendGrid-batchsource.md | 26 +- pom.xml | 30 +- .../sendgrid/common/APIResponseType.java | 38 +++ .../sendgrid/common/SendGridClient.java | 254 ++++++++++++++++ .../common/config/BaseSourceConfig.java | 274 ++++++++++++++++++ .../sendgrid/common/helpers/BaseObject.java | 46 +++ .../sendgrid/common/helpers/EmptyObject.java | 32 ++ .../sendgrid/common/helpers/IBaseObject.java | 42 +++ .../sendgrid/common/helpers/MultiObject.java | 43 +++ .../common/helpers/ObjectDefinition.java | 73 +++++ .../common/helpers/ObjectFieldDefinition.java | 41 +++ .../common/helpers/ObjectFieldInfo.java | 45 +++ .../sendgrid/common/helpers/ObjectHelper.java | 268 +++++++++++++++++ .../sendgrid/common/helpers/ObjectInfo.java | 88 ++++++ .../common/objects/BasicMetadata.java | 31 ++ .../sendgrid/common/objects/BasicResult.java | 49 ++++ .../common/objects/DataSourceGroupType.java | 46 +++ .../common/objects/SendGridAuthType.java | 31 ++ .../common/objects/stats/AdvancedStats.java | 3 +- .../common/objects/stats/CategoryStats.java | 5 +- .../common/objects/stats/GlobalStats.java | 5 +- .../source/batch/SendGridBatchSource.java | 97 +++++++ .../batch/SendGridBatchSourceConfig.java | 27 ++ .../source/batch/SendGridInputFormat.java | 54 ++++ .../batch/SendGridInputFormatProvider.java | 48 +++ .../batch/SendGridMultiRecordReader.java | 102 +++++++ .../source/batch/SendGridRecordReader.java | 89 ++++++ .../sendgrid/source/batch/SendGridSplit.java | 52 ++++ .../source/batch/SendGridTransformer.java | 65 +++++ widgets/SendGrid-batchsource.json | 35 ++- 30 files changed, 2003 insertions(+), 36 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/APIResponseType.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/BaseObject.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/EmptyObject.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/IBaseObject.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/MultiObject.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectDefinition.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldDefinition.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldInfo.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectInfo.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicMetadata.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicResult.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/SendGridAuthType.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridSplit.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridTransformer.java diff --git a/docs/SendGrid-batchsource.md b/docs/SendGrid-batchsource.md index 9eed6e3..d7f72f8 100644 --- a/docs/SendGrid-batchsource.md +++ b/docs/SendGrid-batchsource.md @@ -2,21 +2,22 @@ Description ----------- -This plugin used to query SendGrid v3 API. - +Plugin fetches data from SendGrid. SendGrid is a cloud-based service that assists businesses with email delivery. For +the end user SendGrid provides information about existing Marketing Campaigns, Email Analytics, Bounces, Spam Reports. + Properties ---------- ### General **Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. -**Authentication type:** The way, how user would like to be authenticated to the SendGrid acoount +**Authentication type:** The way, how user would like to be authenticated to the SendGrid account **API Key:** The SendGrid API Key taken from the SendGrid account -**Username:** Login name for SendGrid +**Username:** Login name for the SendGrid account -**Password:** Login password for the username specified above +**Password:** Password for the SendGrid account **Data Source Types:** List of data source groups @@ -25,7 +26,7 @@ Available: - Stats Fields - Suppressions Fields -**Data Source:** One of the above sources picked from list +**Data Source:** SendGrid source object Available: - Marketing Campaigns Fields @@ -42,7 +43,18 @@ Available: - Bounces - Global Unsubscribes - Group Unsubscribes -**Data Source Fields:** + +**Data Source Fields:** The list of fields available for the retrieval + +- *Automation:* id, name, status, type, message_count, created_at, updated_at, live_at +- *SingleSends:* id, name, status, created_at, updated_at, +- *Senders:* id, nickname, address, address_2, city, country, state, zip, locked, created_at, updated_at, from(email, name), verified(status, reason), reply_to(name, email) +- *Contacts:* id, first_name, last_name, list_ids, created_at, updated_at, email, Segments, id, name, parent_list_id, created_at, updated_at, sample_updated_at, contacts_count +- *GlobalStats:* date ,blocks ,bounce_drops ,bounces ,clicks ,deferred ,invalid_emails ,opens ,processed ,requests ,spam_report_drops ,spam_reports ,unique_clicks ,unique_opens ,unsubscribe_drops ,unsubscribes +- *CategoryStats:* name, type, date, blocks, bounce_drops, bounces, clicks, deferred, delivered, invalid_emails, opens, processed, requests, spam_report_drops, spam_reports, unique_clicks, unique_opens, unsubscribe_drops, unsubscribes +- *AdvancedStats:* name, type, date, clicks, opens, unique_clicks, unique_opens +- *Bounces:* created, email, reason, status +- *GroupUnsubscribes:* id ,name ,description ,is_default ,last_email_send_at ,unsubscribes **Start Date:** The date in format YYYY-MM-DD, starting from which the data is requested diff --git a/pom.xml b/pom.xml index d2157e8..25d5df8 100644 --- a/pom.xml +++ b/pom.xml @@ -28,13 +28,29 @@ UTF-8 6.1.0-SNAPSHOT - 2.2.0-SNAPSHOT - 2.3.0 - 2.2.0 + 2.8.0 + 2.3.0-SNAPSHOT 2.2.4 1.2 + + + sonatype + https://oss.sonatype.org/content/groups/public + + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots + + + + + sonatype + https://oss.sonatype.org/content/groups/public/ + + + @@ -84,7 +100,7 @@ io.cdap.plugin hydrator-common - ${cdap.plugin.version} + ${hydrator.version} org.apache.hadoop @@ -213,11 +229,7 @@ true - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - + <_exportcontents>io.cdap.plugin.sendgrid.* *;inline=false;scope=compile true lib diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/APIResponseType.java b/src/main/java/io/cdap/plugin/sendgrid/common/APIResponseType.java new file mode 100644 index 0000000..2b853a7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/APIResponseType.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sendgrid.common; + +/** + * The way, how SendGrid API return the objects + * + */ +public enum APIResponseType { + /** + * Objects come to the response as part of the list + */ + LIST, + + /** + * Objects come as part of wrapper, which consists from the metadata and the result + */ + RESULT, + + /** + * Same as {@link APIResponseType#RESULT} + */ + RESULTS +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java new file mode 100644 index 0000000..b867925 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java @@ -0,0 +1,254 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.sendgrid.Method; +import com.sendgrid.Request; +import com.sendgrid.Response; +import com.sendgrid.SendGrid; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import io.cdap.plugin.sendgrid.common.objects.BasicResult; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; +import io.cdap.plugin.sendgrid.source.batch.SendGridBatchSourceConfig; + +import java.io.IOException; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.annotation.Nullable; + +/** + * SendGrid Client + */ +public class SendGridClient { + + /** + * Extended version of the original SendGrid API wrapper with added support of basic auth + */ + private static class SendGridAPIClient extends SendGrid { + + SendGridAPIClient(String apiKey) { + super(apiKey); + } + + SendGridAPIClient(String username, String password) { + super(""); // actual key is not required due to it would be rewritten with basic auth data + initializeSendGrid(username, password); + } + + /** + * Replaces "Authorization" header, configured for the bearer auth with basic auth credentials + * + * @param username name of the user + * @param password password of the user + */ + private void initializeSendGrid(String username, String password) { + String encoding = Base64.getEncoder().encodeToString((String.format("%s:%s", username, password).getBytes())); + addRequestHeader("Authorization", String.format("Basic %s", encoding)); + } + } + + private SendGridAPIClient sendGrid; + private Gson gson; + + private SendGridClient() { + gson = new GsonBuilder().create(); + } + + public SendGridClient(SendGridBatchSourceConfig config) { + this(); + if (config.getAuthType() == SendGridAuthType.API) { + sendGrid = new SendGridAPIClient(config.getSendGridApiKey()); + } else if (config.getAuthType() == SendGridAuthType.BASIC) { + sendGrid = new SendGridAPIClient(config.getAuthUserName(), config.getAuthPassword()); + } else { + throw new IllegalArgumentException(String.format("Invalid authentication method '%s'", + SendGridClient.class.getCanonicalName())); + } + } + + public SendGridClient(String key) { + this(); + sendGrid = new SendGridAPIClient(key); + } + + public SendGridClient(String username, String password) { + this(); + sendGrid = new SendGridAPIClient(username, password); + } + + /** + * Low level function to query API endpoints + * + * @param method the HTTP method to use + * @param endpoint relative uri to be queried + * @param arguments arguments for the query + * @return query body + * @throws IOException if any issue with query the API happen + */ + private String makeApiRequest(Method method, String endpoint, @Nullable Map arguments) + throws IOException { + + Request request = new Request(); + request.setMethod(method); + request.setEndpoint(endpoint); + + if (arguments != null && !arguments.isEmpty()) { + arguments.forEach(request::addQueryParam); + } + + Response response; + try { + response = sendGrid.api(request); + } catch (IOException e) { + throw new IOException(String.format("Request to SendGrid API \"%s\"", endpoint), e); + } + + return response.getBody(); + } + + /** + * Verify all incoming arguments for the query object + * + * @param objectInfo objects definition + * @param arguments query arguments + * @throws IllegalArgumentException if any validation issue + */ + private void checkIncomingArguments(ObjectInfo objectInfo, Map arguments) + throws IllegalArgumentException { + + if (objectInfo.getRequiredArguments() != null && !objectInfo.getRequiredArguments().isEmpty()) { + if (arguments == null || arguments.isEmpty()) { + throw new IllegalArgumentException(String.format( + "Object '%s' require input arguments to be passed, nothing found", + objectInfo.getCdapObjectName() + )); + } + List exceptions = new ArrayList<>(); + + objectInfo.getRequiredArguments().forEach(x -> { + try { + if (Strings.isNullOrEmpty(x)) { + return; + } + + arguments.keySet().stream() + .filter(x::equals) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(String.format( + "Object '%s' require %s argument, but nothing provided", + objectInfo.getCdapObjectName(), + x + ))); + } catch (IllegalArgumentException e) { + exceptions.add(e.getMessage()); + } + }); + if (!exceptions.isEmpty()) { + throw new IllegalArgumentException(exceptions.stream().collect(Collectors.joining(System.lineSeparator()))); + } + } + } + + /** + * Query SendGrid API using plugin meta objects + * + * @param objectInfo objects definition + * @param arguments query arguments + * @return object representation of the query + * @throws IOException if any issue with query the API happen + * @throws IllegalStateException unsupported response type caught + * @throws IllegalArgumentException if any validation issue + */ + public List getObject(ObjectInfo objectInfo, Map arguments) + throws IOException, IllegalStateException, IllegalArgumentException { + checkIncomingArguments(objectInfo, arguments); + + String endpoint = objectInfo.getSendGridAPIUrl(); + String response = makeApiRequest(Method.GET, endpoint, arguments); + Class clazz = objectInfo.getObjectClass(); + + if (objectInfo.getApiResponseType() == APIResponseType.RESULT) { + Type typeToken = new ParameterizedType() { + @Override + public Type[] getActualTypeArguments() { + return new Type[] { clazz }; + } + + @Override + public Type getRawType() { + return BasicResult.class; + } + + @Override + public Type getOwnerType() { + return null; + } + }; + BasicResult result = gson.fromJson(response, typeToken); + + return result.getResult(); + } else if (objectInfo.getApiResponseType() == APIResponseType.LIST) { + Type typeToken = new ParameterizedType() { + @Override + public Type[] getActualTypeArguments() { + return new Type[] { clazz }; + } + + @Override + public Type getRawType() { + return List.class; + } + + @Override + public Type getOwnerType() { + return null; + } + }; + + return gson.fromJson(response, typeToken); + } else { + throw new IllegalStateException(String.format( + "Unsupported API Response type: %s", + objectInfo.getApiResponseType().name() + )); + } + } + + /** + * Query SendGrid API using plugin meta objects + * + * @param objectInfo objects definition + * @return object representation of the query + * @throws IOException if any issue with query the API happen + * @throws IllegalStateException unsupported response type caught + * @throws IllegalArgumentException if any validation issue + */ + public List getObject(ObjectInfo objectInfo) + throws IOException, IllegalStateException, IllegalArgumentException { + + return getObject(objectInfo, null); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java new file mode 100644 index 0000000..6e59bcf --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java @@ -0,0 +1,274 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.config; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Provides all required configuration for reading SendGrid information + */ +public class BaseSourceConfig extends ReferencePluginConfig { + public static final String PROPERTY_AUTH_TYPE = "authType"; + public static final String PROPERTY_SENDGRID_API_KEY = "sendGridApiKey"; + public static final String PROPERTY_AUTH_USERNAME = "username"; + public static final String PROPERTY_AUTH_PASSWORD = "password"; + public static final String PROPERTY_DATA_SOURCE_TYPES = "dataSourceTypes"; + public static final String PROPERTY_DATA_SOURCE = "dataSource"; + public static final String PROPERTY_DATA_SOURCE_MARKETING = PROPERTY_DATA_SOURCE + "Marketing"; + public static final String PROPERTY_DATA_SOURCE_STATS = PROPERTY_DATA_SOURCE + "Stats"; + public static final String PROPERTY_DATA_SOURCE_SUPPRESSIONS = PROPERTY_DATA_SOURCE + "Suppressions"; + + public static final String PROPERTY_DATA_SOURCE_FIELDS = "dataSourceFields"; + public static final String PROPERTY_STAT_CATEGORIES = "statCategories"; + public static final String PROPERTY_START_DATE = "startDate"; + public static final String PROPERTY_END_DATE = "endDate"; + + @Name(PROPERTY_AUTH_TYPE) + @Description("The way, how user would like to be authenticated to the SendGrid account") + @Macro + private String authType; + + @Name((PROPERTY_SENDGRID_API_KEY)) + @Description("The SendGrid API Key taken from the SendGrid account") + @Macro + @Nullable + private String sendGridApiKey; + + @Name(PROPERTY_AUTH_USERNAME) + @Description("Login name for the SendGrid account") + @Macro + @Nullable + private String authUserName; + + @Name(PROPERTY_AUTH_PASSWORD) + @Description("Password for the SendGrid account") + @Macro + @Nullable + private String authPassword; + + @Name(PROPERTY_DATA_SOURCE_TYPES) + @Description("List of data source groups") + @Macro + private String dataSourceTypes; + + @Name(PROPERTY_DATA_SOURCE_MARKETING) + @Description("SendGrid source objects for the Marketing group") + @Macro + @Nullable + private String dataSourceMarketing; + + @Name(PROPERTY_DATA_SOURCE_STATS) + @Description("SendGrid source objects for the Statistics group") + @Macro + @Nullable + private String dataSourceStats; + + @Name(PROPERTY_DATA_SOURCE_SUPPRESSIONS) + @Description("SendGrid source objects for the Suppressions group") + @Macro + @Nullable + private String dataSourceSuppressions; + + @Name(PROPERTY_DATA_SOURCE_FIELDS) + @Description("The list of fields available for the retrieval") + @Macro + @Nullable + private String dataSourceFields; + + @Name(PROPERTY_START_DATE) + @Description("The date in format YYYY-MM-DD, starting from which the data is requested") + @Nullable + @Macro + private String startDate; + + @Name(PROPERTY_END_DATE) + @Description("The date in format YYYY-MM-DD, the end date for the requested data") + @Nullable + @Macro + private String endDate; + + @Name(PROPERTY_STAT_CATEGORIES) + @Description("List of requested categories for the CategoryStats request") + @Nullable + @Macro + private String statCategories; + + private transient Schema schema; + private transient List dataSource; + private transient Boolean multiObjectMode; + + /** + * Constructor + * + * @param referenceName uniquely identify source/sink for lineage, annotating metadata, etc. + */ + public BaseSourceConfig(String referenceName) { + super(referenceName); + } + + /** + * Fetches all fields selected by the user + */ + public List getFields() { + if (Strings.isNullOrEmpty(dataSourceFields)) { + return Collections.emptyList(); + } + + return Arrays.asList(dataSourceFields.split(",")); + } + + /** + * Aggregates categorized data source + */ + public List getDataSource() { + if (dataSource == null) { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + + if (!Strings.isNullOrEmpty(dataSourceMarketing)) { + builder.add(dataSourceMarketing); + } + if (!Strings.isNullOrEmpty(dataSourceStats)) { + builder.add(dataSourceStats); + } + if (!Strings.isNullOrEmpty(dataSourceSuppressions)) { + builder.add(dataSourceSuppressions); + } + + dataSource = Arrays.asList(String.join(",", builder.build()).split(",")); + } + return dataSource; + } + + /** + * Plugin work mode + */ + public boolean isMultiObjectMode() { + if (multiObjectMode == null) { + multiObjectMode = getDataSource().size() > 1; + } + return multiObjectMode; + } + + /** + * Generated schema according to user configuration + * + * @return user configured schema + */ + public Schema getSchema() { + if (schema == null) { + schema = ObjectHelper.buildSchema(getDataSource(), getFields()); + } + return schema; + } + + /** + * Generates limited schema for mentioned in {@code dataSource} sources + * + * @param dataSource sources to be added to the schema + * + * @return custom schema + */ + public Schema getSchema(List dataSource) { + return ObjectHelper.buildSchema(dataSource, getFields(), isMultiObjectMode()); + } + + /** + * Returns query properties required for some SendGrid Objects + * marked with {@link ObjectDefinition#RequiredArguments()} + */ + public Map getRequestArguments() { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + + if (!Strings.isNullOrEmpty(startDate)) { + builder.put(PROPERTY_START_DATE, startDate); + } + if (!Strings.isNullOrEmpty(endDate)) { + builder.put(PROPERTY_END_DATE, endDate); + } + if (!Strings.isNullOrEmpty(statCategories)) { + builder.put(PROPERTY_END_DATE, statCategories); + } + return builder.build(); + } + + public void validate(FailureCollector failureCollector) { + + } + + /** + * Client authentication way + */ + public SendGridAuthType getAuthType() { + switch (authType) { + case "api": + return SendGridAuthType.API; + case "basic": + return SendGridAuthType.BASIC; + default: + throw new IllegalArgumentException(String.format("Authentication using '%s' is not supported", authType)); + } + } + + /** + * Retrieves Api Key + */ + public String getSendGridApiKey() { + return sendGridApiKey; + } + + /** + * Retrieves username + */ + public String getAuthUserName() { + return authUserName; + } + + /** + * Retrieves password + */ + public String getAuthPassword() { + return authPassword; + } + + @Nullable + public String getStartDate() { + return startDate; + } + + @Nullable + public String getEndDate() { + return endDate; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/BaseObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/BaseObject.java new file mode 100644 index 0000000..1e6e6e2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/BaseObject.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.data.schema.Schema; + +import java.util.Map; + + +/** + * Base object for the {@link IBaseObject} interface + */ +public abstract class BaseObject implements IBaseObject { + + /** + * Return Map of fields according to provided schema + * + * @param schema object schema + * @return fields map + */ + @Override + public Map asFilteredMap(Schema schema) { + ImmutableMap.Builder fields = new ImmutableMap.Builder<>(); + Map allFields = asMap(); + + schema.getFields().stream().map(Schema.Field::getName).forEach(name -> { + fields.put(name, allFields.getOrDefault(name, new EmptyObject())); + }); + + return fields.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/EmptyObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/EmptyObject.java new file mode 100644 index 0000000..d31b2a3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/EmptyObject.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import java.util.Collections; +import java.util.Map; + +/** + * Provides empty stub object + */ +public class EmptyObject extends BaseObject implements IBaseObject { + /** + * Map of all object fields with values + */ + @Override + public Map asMap() { + return Collections.emptyMap(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/IBaseObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/IBaseObject.java new file mode 100644 index 0000000..4c5fa8a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/IBaseObject.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.cdap.api.data.schema.Schema; + +import java.util.Map; + +/** + * Interface for all SendGrid objects + * + * Any child, implemented current interface should provide all + * fields outside through {@link IBaseObject#asMap()} method. + * + * No getters allowed, unless any custom object usage planed. + */ +public interface IBaseObject { + /** + * Map of all object fields with values + */ + Map asMap(); + + /** + * Provide access to object fields mentioned in {@link Schema} + * + * @param schema generated or customized schema for the object + */ + Map asFilteredMap(Schema schema); +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/MultiObject.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/MultiObject.java new file mode 100644 index 0000000..47e59fd --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/MultiObject.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import java.util.HashMap; +import java.util.Map; + +/** + * Multi-Object Holder + */ +public class MultiObject extends BaseObject implements IBaseObject { + + private Map objects; + + public MultiObject() { + objects = new HashMap<>(); + } + + public void addObject(String name, IBaseObject object) { + objects.put(name, object); + } + + /** + * Map of all object fields with values + */ + @Override + public Map asMap() { + return objects; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectDefinition.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectDefinition.java new file mode 100644 index 0000000..8b8356a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectDefinition.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Defines Entity Object with all related options which defines entity + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface ObjectDefinition { + /** + * Object definition type + */ + enum ObjectDefinitionType { + /** + * Describes top-level object + */ + BASE, + + /** + * Describes object, designed to be used as part of another objects + */ + NESTED + } + + /** + * Entity internal name + */ + String Name() default ""; + + /** + * One of the {@link DataSourceGroupType} + */ + DataSourceGroupType Group() default DataSourceGroupType.Marketing; + + /** + * Relative API URI for the Entity + */ + String APIUrl() default ""; + + /** + * The way how REST API provides the information + */ + APIResponseType APIResponseType() default APIResponseType.LIST; + + /** + * List of argument names, required to made request + */ + String[] RequiredArguments() default ""; + + ObjectDefinitionType ObjectType() default ObjectDefinitionType.BASE; +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldDefinition.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldDefinition.java new file mode 100644 index 0000000..d8fdf50 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldDefinition.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.cdap.api.data.schema.Schema; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Defines entity fields + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface ObjectFieldDefinition { + + Schema.Type FieldType() default Schema.Type.STRING; + + /** + * Internal name of the {@link ObjectDefinition#Name()} + *

+ * Only objects with type {@link io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition.ObjectDefinitionType#NESTED} + * allowed + */ + String NestedClass() default ""; +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldInfo.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldInfo.java new file mode 100644 index 0000000..68431a6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectFieldInfo.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.cdap.api.data.schema.Schema; + +/** + * Entity meta-info holder for the {@link ObjectFieldDefinition} annotation + */ +public class ObjectFieldInfo { + private String name; + private Schema.Type type; + private String nestedClass; + + public ObjectFieldInfo(String name, Schema.Type type, String nestedClass) { + this.name = name; + this.type = type; + this.nestedClass = nestedClass; + } + + public String getName() { + return name; + } + + public Schema.Type getType() { + return type; + } + + public String getNestedClassName() { + return nestedClass; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java new file mode 100644 index 0000000..55b94db --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java @@ -0,0 +1,268 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingAutomation; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingContacts; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSegments; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSenders; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSendersContact; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSendersVerified; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingSingleSend; +import io.cdap.plugin.sendgrid.common.objects.stats.AdvancedStats; +import io.cdap.plugin.sendgrid.common.objects.stats.CategoryStats; +import io.cdap.plugin.sendgrid.common.objects.stats.GlobalStats; +import io.cdap.plugin.sendgrid.common.objects.stats.MetricStats; +import io.cdap.plugin.sendgrid.common.objects.stats.StatsStats; +import io.cdap.plugin.sendgrid.common.objects.suppressions.BounceSuppression; +import io.cdap.plugin.sendgrid.common.objects.suppressions.GlobalUnsubscribeSuppression; +import io.cdap.plugin.sendgrid.common.objects.suppressions.GroupUnsubscribeSuppression; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.Nullable; + +/** + * Schema processing core. Deals with such work as: + * - resolves schema from annotated entities + * - builds custom schemas with accounting base schema structure + * - support nesting entities, maps + * ToDo: add list support + */ +public class ObjectHelper { + + /* + Could be used Reflections to discover the objects, but it is additional + dependency and time to dynamically discover them + */ + private static List objects = Arrays.asList( + // base objects + MarketingAutomation.class, + MarketingContacts.class, + MarketingSegments.class, + MarketingSenders.class, + MarketingSingleSend.class, + AdvancedStats.class, + CategoryStats.class, + GlobalStats.class, + BounceSuppression.class, + GlobalUnsubscribeSuppression.class, + GroupUnsubscribeSuppression.class, + + // nested objects + MarketingSendersContact.class, + MarketingSendersVerified.class, + MetricStats.class, + StatsStats.class + ); + + private static Map objectsDefinitions; + + static { + // resolves available entities schema on first access + buildSchemaDefinition(); + } + + /** + * Create schema definition for annotated objects + */ + private static void buildSchemaDefinition() { + if (objectsDefinitions != null) { + return; + } + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + objects.forEach(object -> { + try { + ObjectDefinition objectDefinition = (ObjectDefinition) object.getAnnotation(ObjectDefinition.class); + + List objectFieldInfos = Arrays.stream(object.getDeclaredFields()).map(x -> { + try { + String name = x.getAnnotation(SerializedName.class).value(); + ObjectFieldDefinition objectFieldDefinition = x.getAnnotation(ObjectFieldDefinition.class); + + return new ObjectFieldInfo(name, objectFieldDefinition.FieldType(), objectFieldDefinition.NestedClass()); + } catch (NullPointerException e) { + return null; // Ignore non-annotated fields + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + + builder.put(object.getName(), new ObjectInfo( + objectDefinition.Name(), + objectFieldInfos, + objectDefinition.APIUrl(), + objectDefinition.APIResponseType(), + object, + objectDefinition.Group(), + Arrays.asList(objectDefinition.RequiredArguments()), + objectDefinition.ObjectType() + )); + + } catch (NullPointerException e) { + throw new RuntimeException(String.format("Object with name %s not annotated with %s", object.getName(), + ObjectDefinition.class.getName())); + } + }); + objectsDefinitions = builder.build(); + } + + public static List getObjectNames() { + return objectsDefinitions.values().stream() + .filter(x -> x.getObjectType() == ObjectDefinition.ObjectDefinitionType.BASE) + .map(ObjectInfo::getCdapObjectName) + .collect(Collectors.toList()); + } + + /** + * Gets stream for {@link ObjectDefinition.ObjectDefinitionType#BASE} objects + */ + public static Stream getObjectStream() { + return objects.stream() + .filter(x -> getObjectInfo(x).getObjectType() == ObjectDefinition.ObjectDefinitionType.BASE); + } + + /** + * Gets stream for {@link ObjectDefinition.ObjectDefinitionType#NESTED} objects + */ + public static Stream getNestedObjectStream() { + return objects.stream() + .filter(x -> getObjectInfo(x).getObjectType() == ObjectDefinition.ObjectDefinitionType.NESTED); + } + + /** + * Provides entity schema definition + * + * @param objectClass entity class, which derived from {@link IBaseObject} + */ + public static ObjectInfo getObjectInfo(Class objectClass) { + return objectsDefinitions.get(objectClass.getName()); + } + + /** + * Provides entity schema definition + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + */ + @Nullable + public static ObjectInfo getObjectInfo(String internalObjectName) { + return objectsDefinitions.values().stream() + .filter(x -> x.getCdapObjectName().equals(internalObjectName)) + .findFirst() + .orElse(null); + } + + /** + * Provides schema definition for desired entities with only desired fields included + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + * @param requestedFields the names, provided via {@link SerializedName#value()} + * + * @return CDAP Schema + */ + public static Schema buildSchema(List internalObjectName, @Nullable List requestedFields) { + return buildSchema(internalObjectName, requestedFields, false); + } + + /** + * Provides schema definition for desired entities with only desired fields included + * + * For single requested object: + * Schema generator creates plain schema, where each field represents as column + * and whole schema describes only one entity + * + * For multi-object request: + * Schema generator creates additional top-level holder, where each column represents + * separate requested entity + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + * @param requestedFields the names, provided via {@link SerializedName#value()} + * @param alwaysMultiObject regulates how to generate schema if only one entity is requested + * + * @return CDAP Schema + */ + public static Schema buildSchema(List internalObjectName, @Nullable List requestedFields, + boolean alwaysMultiObject) { + // generate simple schema for the single object + if (!alwaysMultiObject && internalObjectName.size() == 1) { + return buildSchema(internalObjectName.get(0), requestedFields); + } + + List fields = new ArrayList<>(); + + internalObjectName.forEach(object -> { + List objectFields = buildSchema(object, requestedFields).getFields(); + Schema.Field field = Schema.Field.of(object, Schema.recordOf(object, Objects.requireNonNull(objectFields))); + + fields.add(field); + }); + + return Schema.recordOf("output", fields); + } + + /** + * Provides schema definition for desired entity with only desired fields included + * + * @param internalObjectName the name, provided via {@link ObjectDefinition#Name()} + * @param requestedFields the names, provided via {@link SerializedName#value()} + * + * @return CDAP Schema + */ + public static Schema buildSchema(String internalObjectName, @Nullable List requestedFields) { + ObjectInfo objectInfo = getObjectInfo(internalObjectName); + + if (objectInfo == null) { + return Schema.recordOf(internalObjectName); + } + + List fieldInfos; + + if (requestedFields == null || requestedFields.isEmpty()) { + fieldInfos = objectInfo.getFieldDefinitions(); + } else { + fieldInfos = objectInfo.getFieldsDefinitions(requestedFields); + } + + List cdapFields = fieldInfos.stream() + .map(x -> { + if (x.getType() == Schema.Type.MAP) { + if (Strings.isNullOrEmpty(x.getNestedClassName())) { + throw new IllegalArgumentException(String.format("Nested class is not declared for the field %s", + x.getName())); + } + List nestedFields = buildSchema(x.getNestedClassName(), requestedFields).getFields(); + + return Schema.Field.of(x.getName(), + Schema.recordOf(x.getName(), Objects.requireNonNull(nestedFields)) + ); + } + return Schema.Field.of(x.getName(), Schema.nullableOf(Schema.of(x.getType()))); + }) + .collect(Collectors.toList()); + + return (cdapFields.isEmpty()) + ? Schema.recordOf(objectInfo.getCdapObjectName()) + : Schema.recordOf(objectInfo.getCdapObjectName(), cdapFields); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectInfo.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectInfo.java new file mode 100644 index 0000000..d6b6047 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectInfo.java @@ -0,0 +1,88 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.helpers; + +import io.cdap.plugin.sendgrid.common.APIResponseType; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Entity meta-info holder for {@link ObjectDefinition} + */ +public class ObjectInfo { + private String cdapObjectName; + + private String sendGridAPIUrl; + private DataSourceGroupType dataSourceGroupType; + private Class objectClass; + private List fieldDefinitions; + private APIResponseType apiResponseType; + private List requiredArguments; + private ObjectDefinition.ObjectDefinitionType objectType; + + public ObjectInfo(String cdapObjectName, List fieldDefinitions, String sendGridAPIUrl, + APIResponseType apiResponseType, Class objectClass, DataSourceGroupType dataSourceGroupType, + List requiredArguments, ObjectDefinition.ObjectDefinitionType objectType) { + this.cdapObjectName = cdapObjectName; + this.fieldDefinitions = fieldDefinitions; + this.sendGridAPIUrl = sendGridAPIUrl; + this.apiResponseType = apiResponseType; + this.objectClass = objectClass; + this.dataSourceGroupType = dataSourceGroupType; + this.requiredArguments = requiredArguments; + this.objectType = objectType; + } + + public String getCdapObjectName() { + return cdapObjectName; + } + + public String getSendGridAPIUrl() { + return sendGridAPIUrl; + } + + public List getFieldDefinitions() { + return fieldDefinitions; + } + + public List getFieldsDefinitions(List fields) { + return fieldDefinitions.stream() + .filter(x -> fields.stream().anyMatch(y -> x.getName().equals(y))) + .collect(Collectors.toList()); + } + + public Class getObjectClass() { + return objectClass; + } + + public DataSourceGroupType getDataSourceGroupType() { + return dataSourceGroupType; + } + + public APIResponseType getApiResponseType() { + return apiResponseType; + } + + public List getRequiredArguments() { + return requiredArguments.stream().filter(x -> !x.equals("")).collect(Collectors.toList()); + } + + public ObjectDefinition.ObjectDefinitionType getObjectType() { + return objectType; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicMetadata.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicMetadata.java new file mode 100644 index 0000000..c860a0b --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicMetadata.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects; + +import com.google.gson.annotations.SerializedName; + +/** + * SendGrid API Response Wrapper + */ +public class BasicMetadata { + + @SerializedName("self") + private String objectUrl; + + public String getSelfUrl() { + return objectUrl; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicResult.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicResult.java new file mode 100644 index 0000000..c3f8b32 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/BasicResult.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sendgrid.common.objects; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * SendGrid API Response Wrapper + * + * @param Any {@link io.cdap.plugin.sendgrid.common.helpers.BaseObject} object + */ +public class BasicResult { + @SerializedName(value = "result") + private List result; + + @SerializedName(value = "results") + private List results; + + @Nullable + @SerializedName("_metadata") + private BasicMetadata metadata; + + public List getResult() { + return (results == null) ? result : results; + } + + @Nullable + public BasicMetadata getMetadata() { + return metadata; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java new file mode 100644 index 0000000..5142c3d --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects; + +import java.util.Arrays; + +/** + * Entities groups + */ +public enum DataSourceGroupType { + Marketing("Marketing Campaign Objects"), + Stats("Statistic Objects"), + Suppressions("Suppression Objects"); + + private String value; + + private static String className = DataSourceGroupType.class.getName(); + + DataSourceGroupType(String dataSourceType) { + this.value = dataSourceType; + } + + public static DataSourceGroupType fromString(String value) { + return Arrays.stream(DataSourceGroupType.values()) + .filter(group -> group.value.equals(value)) + .findFirst() + .orElseThrow(() -> new IllegalStateException(String.format("'%s' is invalid %s", value, className))); + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/SendGridAuthType.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/SendGridAuthType.java new file mode 100644 index 0000000..71c05cf --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/SendGridAuthType.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects; + +/** + * SendGrid authentication way + */ +public enum SendGridAuthType { + /** + * API KEY + */ + API, + + /** + * HTTP Basic auth + */ + BASIC +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java index a1dbcee..48a78a4 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; @@ -36,7 +37,7 @@ Group = DataSourceGroupType.Stats, APIUrl = "geo/stats", RequiredArguments = { - "start_date" // argument format: YYYY-MM-DD + BaseSourceConfig.PROPERTY_START_DATE } ) public class AdvancedStats extends BaseObject implements IBaseObject { diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java index 73a21d5..b95aee7 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; @@ -36,8 +37,8 @@ Group = DataSourceGroupType.Stats, APIUrl = "categories/stats", RequiredArguments = { - "start_date", // argument format: YYYY-MM-DD - "categories" + BaseSourceConfig.PROPERTY_START_DATE, + BaseSourceConfig.PROPERTY_STAT_CATEGORIES } ) public class CategoryStats extends BaseObject implements IBaseObject { diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java index 52a26e6..8d30c11 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; @@ -35,7 +36,9 @@ Name = "GlobalStats", Group = DataSourceGroupType.Stats, APIUrl = "stats", - RequiredArguments = {"start_date"} + RequiredArguments = { + BaseSourceConfig.PROPERTY_START_DATE + } ) public class GlobalStats extends BaseObject implements IBaseObject { diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java new file mode 100644 index 0000000..11c82b3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import com.google.common.base.Preconditions; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import org.apache.hadoop.io.NullWritable; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Batch Source plugin + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(SendGridBatchSource.NAME) +@Description("Reads data from SendGrid API") +public class SendGridBatchSource extends BatchSource { + public static final String NAME = "SendGrid"; + + private final SendGridBatchSourceConfig config; + + public SendGridBatchSource(SendGridBatchSourceConfig config) { + this.config = config; + } + + @Override + public void prepareRun(BatchSourceContext batchSourceContext) throws Exception { + validateConfiguration(batchSourceContext.getFailureCollector()); + + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading SendGrid Objects", + Preconditions.checkNotNull(config.getSchema().getFields()) + .stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new SendGridInputFormatProvider(config))); + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); + + IdUtils.validateReferenceName(config.referenceName, failureCollector); + validateConfiguration(failureCollector); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void transform(KeyValue input, Emitter emitter) { + Schema schema; + if (config.isMultiObjectMode()) { + List fetchedDataSources = new ArrayList<>(input.getValue().asMap().keySet()); + schema = config.getSchema(fetchedDataSources); + } else { + schema = config.getSchema(); + } + emitter.emit(SendGridTransformer.transform(input.getValue(), schema)); + } + + @SuppressWarnings("ThrowableNotThrown") + private void validateConfiguration(FailureCollector failureCollector) { + config.validate(failureCollector); + failureCollector.getOrThrowException(); + } + +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSourceConfig.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSourceConfig.java new file mode 100644 index 0000000..2663a8b --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSourceConfig.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; + +/** + * SendGrid Source Plugin configuration + */ +public class SendGridBatchSourceConfig extends BaseSourceConfig { + public SendGridBatchSourceConfig(String referenceName) { + super(referenceName); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java new file mode 100644 index 0000000..f5ac25f --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * SendGrid InputFormat + */ +public class SendGridInputFormat extends InputFormat { + private static final Gson gson = new GsonBuilder().create(); + + @Override + public List getSplits(JobContext context) throws IOException, InterruptedException { + return Collections.singletonList(new SendGridSplit()); + } + + @Override + public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) + throws IOException, InterruptedException { + + Configuration conf = context.getConfiguration(); + String serializedConfig = conf.get(SendGridInputFormatProvider.PROPERTY_CONFIG_JSON); + SendGridBatchSourceConfig sgConfig = gson.fromJson(serializedConfig, SendGridBatchSourceConfig.class); + + return (sgConfig.isMultiObjectMode()) + ? new SendGridMultiRecordReader() + : new SendGridRecordReader(); + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java new file mode 100644 index 0000000..124c07c --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Map; + +/** + * Input stream format provider + */ +public class SendGridInputFormatProvider implements InputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.sendgrid.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + SendGridInputFormatProvider(SendGridBatchSourceConfig config) { + this.conf = new ImmutableMap.Builder() + .put(PROPERTY_CONFIG_JSON, gson.toJson(config)) + .build(); + } + + @Override + public String getInputFormatClassName() { + return SendGridInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java new file mode 100644 index 0000000..0d0b009 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java @@ -0,0 +1,102 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.MultiObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * SendGrid MultiRecord Reader + */ +public class SendGridMultiRecordReader extends RecordReader { + private static final Gson gson = new GsonBuilder().create(); + + private Map> recordIterators; + private Map currentRecords; + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { + String serializedConfig = context.getConfiguration().get(SendGridInputFormatProvider.PROPERTY_CONFIG_JSON); + SendGridBatchSourceConfig sgConfig = gson.fromJson(serializedConfig, SendGridBatchSourceConfig.class); + SendGridClient client = new SendGridClient(sgConfig); + ImmutableMap.Builder> iterators = new ImmutableMap.Builder<>(); + + sgConfig.getDataSource().forEach(x -> { + ObjectInfo objectInfo = ObjectHelper.getObjectInfo(x); + try { + iterators.put(x, client.getObject(objectInfo, sgConfig.getRequestArguments()).iterator()); + } catch (IOException e) { + iterators.put(x, Collections.emptyIterator()); + } + }); + recordIterators = iterators.build(); + currentRecords = new HashMap<>(); + } + + @Override + public boolean nextKeyValue() throws IOException, InterruptedException { + currentRecords.clear(); + + return recordIterators.entrySet().stream() + .map(k -> { + boolean currHasNext = k.getValue().hasNext(); + if (currHasNext) { + currentRecords.put(k.getKey(), k.getValue().next()); + } + return currHasNext; + }) + .reduce(false, (prev, curr) -> prev || curr); + } + + @Override + public NullWritable getCurrentKey() throws IOException, InterruptedException { + return null; + } + + @Override + public IBaseObject getCurrentValue() throws IOException, InterruptedException { + MultiObject multiObject = new MultiObject(); + currentRecords.forEach(multiObject::addObject); + + return multiObject; + } + + @Override + public float getProgress() throws IOException, InterruptedException { + return 0.0f; + } + + @Override + public void close() throws IOException { + // no-op + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java new file mode 100644 index 0000000..39955e8 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; + +/** + * SendGrid Record Reader + */ +public class SendGridRecordReader extends RecordReader { + private static final Gson gson = new GsonBuilder().create(); + + private Iterator recordIterator; + private IBaseObject currentRecord; + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { + Configuration conf = context.getConfiguration(); + String serializedConfig = conf.get(SendGridInputFormatProvider.PROPERTY_CONFIG_JSON); + SendGridBatchSourceConfig sgConfig = gson.fromJson(serializedConfig, SendGridBatchSourceConfig.class); + + SendGridClient client = new SendGridClient(sgConfig); + + Iterator objectsIterator = sgConfig.getDataSource().iterator(); + if (objectsIterator.hasNext()) { + ObjectInfo currentObject = ObjectHelper.getObjectInfo(objectsIterator.next()); + recordIterator = client.getObject(currentObject, sgConfig.getRequestArguments()).iterator(); + } else { + recordIterator = Collections.emptyIterator(); + } + } + + @Override + public boolean nextKeyValue() throws IOException, InterruptedException { + boolean recordHasNext = recordIterator.hasNext(); + + if (recordHasNext) { + currentRecord = recordIterator.next(); + } + return recordHasNext; + } + + @Override + public NullWritable getCurrentKey() throws IOException, InterruptedException { + return null; + } + + @Override + public IBaseObject getCurrentValue() throws IOException, InterruptedException { + return currentRecord; + } + + @Override + public float getProgress() throws IOException, InterruptedException { + return 0.0f; + } + + @Override + public void close() throws IOException { + // no-op + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridSplit.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridSplit.java new file mode 100644 index 0000000..05afa0c --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridSplit.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A no-op split + */ +public class SendGridSplit extends InputSplit implements Writable { + public SendGridSplit() { + + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + + } + + @Override + public long getLength() throws IOException, InterruptedException { + return 0; + } + + @Override + public String[] getLocations() throws IOException, InterruptedException { + return new String[0]; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridTransformer.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridTransformer.java new file mode 100644 index 0000000..d365525 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridTransformer.java @@ -0,0 +1,65 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sendgrid.common.helpers.EmptyObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; + +import java.util.Map; +import java.util.Objects; + +/** + * Record transformer + */ +public class SendGridTransformer { + + @SuppressWarnings("unchecked") + private static void transformValue(String k, Object v, Schema schema, StructuredRecord.Builder builder) { + + if (v instanceof Map) { + Schema mapSchema = Objects.requireNonNull(schema.getField(k)).getSchema(); + builder.set(k, transform((Map) v, mapSchema)); + } else if (v instanceof EmptyObject) { + // no-op + } else if (v instanceof IBaseObject) { + Schema mapSchema = Objects.requireNonNull(schema.getField(k)).getSchema(); + builder.set(k, transform((IBaseObject) v, mapSchema)); + } else { + builder.set(k, v); + } + } + + public static StructuredRecord transform(Map object, Schema schema) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + object.entrySet().stream() + .filter(k -> schema.getField(k.getKey()) != null) // filter absent fields in the schema + .forEach(k -> transformValue(k.getKey(), k.getValue(), schema, builder)); + + return builder.build(); + } + + public static StructuredRecord transform(IBaseObject object, Schema schema) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + object.asFilteredMap(schema) + .forEach((k, v) -> transformValue(k, v, schema, builder)); + + return builder.build(); + } +} diff --git a/widgets/SendGrid-batchsource.json b/widgets/SendGrid-batchsource.json index 4042bf6..e45e82c 100644 --- a/widgets/SendGrid-batchsource.json +++ b/widgets/SendGrid-batchsource.json @@ -1,10 +1,10 @@ { "metadata": { - "spec-version": "1.0" + "spec-version": "1.5" }, "configuration-groups": [ { - "label": "General", + "label": "Basic", "properties": [ { "name": "referenceName", @@ -33,20 +33,20 @@ { "name": "sendGridApiKey", "label": "API Key", - "widget-type": "textbox", - "default": "" + "widget-type": "securekey-text", + "widget-attributes": { + "placeholder": "SendGrid API Key" + } }, { "name": "username", "label": "Username", - "widget-type": "textbox", - "default": "" + "widget-type": "textbox" }, { "name": "password", "label": "Password", - "widget-type": "password", - "default": "" + "widget-type": "password" }, { "name": "dataSourceTypes", @@ -97,8 +97,7 @@ "label": "Single Sends" } ], - "delimiter": ",", - "default": "" + "delimiter": "," } }, { @@ -136,8 +135,7 @@ "label": "Group Unsubscribes" } ], - "delimiter": ",", - "default": "" + "delimiter": "," } }, { @@ -159,8 +157,7 @@ "label": "Group Unsubscribes" } ], - "delimiter": ",", - "default": "" + "delimiter": "," } }, { @@ -318,13 +315,19 @@ "name": "startDate", "label": "Start Date", "widget-type": "textbox", - "default": "" + "placeholder": "YYYY-MM-DD" }, { "name": "endDate", "label": "End Date", "widget-type": "textbox", - "defaut": "" + "placeholder": "YYYY-MM-DD" + }, + { + "name": "statCategories", + "label": "Statistic Categories", + "widget-type": "textbox", + "placeholder": "spam,..." } ] } From 05cd74240ed395c0133c8310e284105df4855022 Mon Sep 17 00:00:00 2001 From: Dmitry Grinenko Date: Sat, 26 Oct 2019 23:13:14 +0300 Subject: [PATCH 3/6] added validation --- .gitignore | 1 + pom.xml | 20 ++ .../sendgrid/common/SendGridClient.java | 34 ++- ...{BaseSourceConfig.java => BaseConfig.java} | 26 +- .../common/config/BaseConfigValidator.java | 230 ++++++++++++++++++ .../sendgrid/common/helpers/ObjectHelper.java | 15 ++ .../common/objects/DataSourceGroupType.java | 6 +- .../common/objects/stats/AdvancedStats.java | 6 +- .../common/objects/stats/CategoryStats.java | 6 +- .../common/objects/stats/GlobalStats.java | 4 +- .../common/objects/stats/MetricStats.java | 17 ++ .../common/objects/stats/StatsStats.java | 5 + ...ceConfig.java => SendGridBatchConfig.java} | 11 +- .../batch/SendGridBatchConfigValidator.java | 34 +++ .../source/batch/SendGridBatchSource.java | 4 +- .../source/batch/SendGridInputFormat.java | 2 +- .../batch/SendGridInputFormatProvider.java | 2 +- .../batch/SendGridMultiRecordReader.java | 2 +- .../source/batch/SendGridRecordReader.java | 2 +- .../source/batch/SendGridTransformer.java | 13 + widgets/SendGrid-batchsource.json | 28 +-- 21 files changed, 415 insertions(+), 53 deletions(-) rename src/main/java/io/cdap/plugin/sendgrid/common/config/{BaseSourceConfig.java => BaseConfig.java} (92%) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfigValidator.java rename src/main/java/io/cdap/plugin/sendgrid/source/batch/{SendGridBatchSourceConfig.java => SendGridBatchConfig.java} (67%) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchConfigValidator.java diff --git a/.gitignore b/.gitignore index 500cf5a..4e2379d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* +*.xml # project files *.iml .idea \ No newline at end of file diff --git a/pom.xml b/pom.xml index 25d5df8..2b12cfc 100644 --- a/pom.xml +++ b/pom.xml @@ -76,8 +76,28 @@ org.slf4j slf4j-log4j12 + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + 4.5.2 + compile + + + org.apache.httpcomponents + httpcore + 4.4.4 + compile + com.google.code.gson diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java index b867925..1fc6e39 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java @@ -18,6 +18,7 @@ import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; import com.sendgrid.Method; import com.sendgrid.Request; import com.sendgrid.Response; @@ -26,13 +27,14 @@ import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; import io.cdap.plugin.sendgrid.common.objects.BasicResult; import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; -import io.cdap.plugin.sendgrid.source.batch.SendGridBatchSourceConfig; +import io.cdap.plugin.sendgrid.source.batch.SendGridBatchConfig; import java.io.IOException; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -43,6 +45,7 @@ * SendGrid Client */ public class SendGridClient { + private static final String CONNECTION_CHECK_ENDPOINT = "alerts"; /** * Extended version of the original SendGrid API wrapper with added support of basic auth @@ -77,7 +80,7 @@ private SendGridClient() { gson = new GsonBuilder().create(); } - public SendGridClient(SendGridBatchSourceConfig config) { + public SendGridClient(SendGridBatchConfig config) { this(); if (config.getAuthType() == SendGridAuthType.API) { sendGrid = new SendGridAPIClient(config.getSendGridApiKey()); @@ -123,12 +126,37 @@ private String makeApiRequest(Method method, String endpoint, @Nullable Map>> errors = gson.fromJson( + messages[1], + new TypeToken>>>() { }.getType() + ); + if (errors.containsKey("errors")) { + String description = errors.get("errors").stream() + .filter(x -> x.containsKey("message")) + .map(x -> x.get("message")) + .collect(Collectors.joining(";")); + + serverMessage = String.format("%s, API response: %s", messages[0], description); + } + } + throw new IOException(serverMessage, e); } return response.getBody(); } + /** + * Checks connection to the service by testing API endpoint, in case + * of exception would be generated {@link IOException} + */ + public void checkConnection() throws IOException { + makeApiRequest(Method.GET, CONNECTION_CHECK_ENDPOINT, null); + } + /** * Verify all incoming arguments for the query object * diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java similarity index 92% rename from src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java rename to src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java index 6e59bcf..34db4d7 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java @@ -39,7 +39,7 @@ /** * Provides all required configuration for reading SendGrid information */ -public class BaseSourceConfig extends ReferencePluginConfig { +public class BaseConfig extends ReferencePluginConfig { public static final String PROPERTY_AUTH_TYPE = "authType"; public static final String PROPERTY_SENDGRID_API_KEY = "sendGridApiKey"; public static final String PROPERTY_AUTH_USERNAME = "username"; @@ -52,8 +52,8 @@ public class BaseSourceConfig extends ReferencePluginConfig { public static final String PROPERTY_DATA_SOURCE_FIELDS = "dataSourceFields"; public static final String PROPERTY_STAT_CATEGORIES = "statCategories"; - public static final String PROPERTY_START_DATE = "startDate"; - public static final String PROPERTY_END_DATE = "endDate"; + public static final String PROPERTY_START_DATE = "start_date"; + public static final String PROPERTY_END_DATE = "end_date"; @Name(PROPERTY_AUTH_TYPE) @Description("The way, how user would like to be authenticated to the SendGrid account") @@ -134,10 +134,17 @@ public class BaseSourceConfig extends ReferencePluginConfig { * * @param referenceName uniquely identify source/sink for lineage, annotating metadata, etc. */ - public BaseSourceConfig(String referenceName) { + public BaseConfig(String referenceName) { super(referenceName); } + /** + * Validate configuration for the issues + */ + protected void validate(FailureCollector failureCollector) { + new BaseConfigValidator(failureCollector, this).validate(); + } + /** * Fetches all fields selected by the user */ @@ -223,10 +230,6 @@ public Map getRequestArguments() { return builder.build(); } - public void validate(FailureCollector failureCollector) { - - } - /** * Client authentication way */ @@ -271,4 +274,11 @@ public String getStartDate() { public String getEndDate() { return endDate; } + + public List getDataSourceTypes() { + if (!Strings.isNullOrEmpty(dataSourceTypes)) { + return Arrays.asList(dataSourceTypes.split(",")); + } + return Collections.emptyList(); + } } diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfigValidator.java b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfigValidator.java new file mode 100644 index 0000000..7f0c324 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfigValidator.java @@ -0,0 +1,230 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.config; + +import com.google.common.base.Strings; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.validation.ValidationFailure; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.helpers.ObjectInfo; +import io.cdap.plugin.sendgrid.common.objects.DataSourceGroupType; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; + +import java.io.IOException; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Validates configuration + */ +public class BaseConfigValidator { + protected FailureCollector failureCollector; + protected SendGridClient client = null; + private BaseConfig config; + + private static Pattern datePattern = Pattern.compile("(?\\d{4})-(?\\d{2})-(?\\d{1,2})"); + + public BaseConfigValidator(FailureCollector failureCollector, BaseConfig config) { + this.failureCollector = failureCollector; + this.config = config; + } + + private void checkAuthType() { + try { + config.getAuthType(); + } catch (IllegalArgumentException e) { + failureCollector.addFailure(String.format("Wrong authentication method selected: %s", e.getMessage()), null) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_TYPE); + } + } + + private Stream getObjectsStream() { + return config.getDataSource().stream() + .filter(x -> !Strings.isNullOrEmpty(x)) + .map(ObjectHelper::getObjectInfo); + } + + private void checkAuthData() { + boolean tryToLogin = true; + + switch (config.getAuthType()) { + case BASIC: + if (Strings.isNullOrEmpty(config.getAuthUserName())) { + failureCollector.addFailure("User name is not set", null) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_USERNAME); + tryToLogin = false; + } + if (Strings.isNullOrEmpty(config.getAuthPassword())) { + failureCollector.addFailure("Password is not set", null) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_PASSWORD); + tryToLogin = false; + } + + if (tryToLogin) { + client = new SendGridClient(config.getAuthUserName(), config.getAuthPassword()); + } + break; + case API: + if (Strings.isNullOrEmpty(config.getSendGridApiKey())) { + failureCollector.addFailure("API Key is not set", null) + .withConfigProperty(BaseConfig.PROPERTY_SENDGRID_API_KEY); + tryToLogin = false; + } + + if (tryToLogin) { + client = new SendGridClient(config.getSendGridApiKey()); + } + break; + } + } + + private void checkCategoriesSelection() { + if (config.getDataSourceTypes().isEmpty()) { + failureCollector.addFailure("Object categories are not set", null) + .withConfigProperty(BaseConfig.PROPERTY_DATA_SOURCE_TYPES); + } + + config.getDataSourceTypes() + .forEach(x -> { + try { + DataSourceGroupType.fromString(x); + } catch (IllegalStateException e) { + failureCollector.addFailure( + String.format("Unknown '%s' data source type: %s", x , e.getMessage()), null) + .withStacktrace(e.getStackTrace()); + } + }); + } + + private void checkObjectsSelection() { + List objects = getObjectsStream().collect(Collectors.toList()); + List categories = config.getDataSourceTypes().stream() + .map(DataSourceGroupType::fromString) + .collect(Collectors.toList()); + + categories.forEach(category -> { + if (objects.stream().noneMatch(x -> x.getDataSourceGroupType() == category)) { + failureCollector.addFailure( + String.format("No objects selected for the category: %s", category.name()), null) + .withConfigProperty(BaseConfig.PROPERTY_DATA_SOURCE); + } + }); + } + + private void checkFieldSelection() { + List objects = getObjectsStream().collect(Collectors.toList()); + List fields = config.getFields(); + + objects.forEach(object -> { + if (object.getFieldsDefinitions(fields).isEmpty()) { + failureCollector.addFailure( + String.format("No fields selected for object '%s'", object.getCdapObjectName()), null) + .withConfigProperty(BaseConfig.PROPERTY_DATA_SOURCE_FIELDS); + } + }); + } + + private void checkRequiredInputProperties() { + getObjectsStream() + .flatMap(obj -> obj.getRequiredArguments().stream()) + .filter(arg -> !config.getRequestArguments().containsKey(arg)) + .forEach(arg -> { + failureCollector.addFailure( + String.format("Argument %s cannot be empty", arg), null) + .withConfigProperty(arg); + }); + } + + private void checkDateFormat(String date, String field) { + if (!Strings.isNullOrEmpty(date)) { + Matcher matcher = datePattern.matcher(date); + if (!matcher.find()) { + failureCollector.addFailure("Input format should math YYYY-MM-DD", null) + .withConfigProperty(field); + } else { + Integer month; + Integer day; + + try { + month = Integer.valueOf(matcher.group("month")); + day = Integer.valueOf(matcher.group("day")); + } catch (NumberFormatException e) { + failureCollector.addFailure("Input format should math YYYY-MM-DD", null) + .withConfigProperty(field); + return; + } + + if (month < 1 || month > 12) { + failureCollector.addFailure( + "Input format should math YYYY-MM-DD and MM should be in range from 1 to 12", null) + .withConfigProperty(field); + } + + if (day < 1 || day > 31) { + failureCollector.addFailure( + "Input format should math YYYY-MM-DD and DD should be in range from 1 to 31", null) + .withConfigProperty(field); + } + } + } + } + + private void checkDateArguments() { + checkDateFormat(config.getStartDate(), BaseConfig.PROPERTY_START_DATE); + checkDateFormat(config.getEndDate(), BaseConfig.PROPERTY_END_DATE); + } + + private void checkClientConnectivity() { + try { + client.checkConnection(); + } catch (IOException e) { + ValidationFailure failure = failureCollector + .addFailure(String.format("Connectivity issues: %s", e.getMessage()), null) + .withStacktrace(e.getStackTrace()); + + if (config.getAuthType() == SendGridAuthType.BASIC) { + failure + .withConfigProperty(BaseConfig.PROPERTY_AUTH_USERNAME) + .withConfigProperty(BaseConfig.PROPERTY_AUTH_PASSWORD); + } + if (config.getAuthType() == SendGridAuthType.API) { + failure.withConfigProperty(BaseConfig.PROPERTY_SENDGRID_API_KEY); + } + } + } + + public void validate() { + client = null; + + checkAuthType(); + checkAuthData(); + checkCategoriesSelection(); + checkObjectsSelection(); + checkFieldSelection(); + checkRequiredInputProperties(); + checkDateArguments(); + + if (client != null) { // client could be not constructed, if any of checkAuth tests failed + checkClientConnectivity(); + } + } + + +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java index 55b94db..c965a1c 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java @@ -242,6 +242,9 @@ public static Schema buildSchema(String internalObjectName, @Nullable List cdapFields = fieldInfos.stream() @@ -257,6 +260,18 @@ public static Schema buildSchema(String internalObjectName, @Nullable List nestedFields = buildSchema(x.getNestedClassName(), requestedFields).getFields(); + return Schema.Field.of(x.getName(), + Schema.arrayOf( + Schema.recordOf(x.getName(), Objects.requireNonNull(nestedFields)) + ) + ); + } return Schema.Field.of(x.getName(), Schema.nullableOf(Schema.of(x.getType()))); }) .collect(Collectors.toList()); diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java index 5142c3d..af755d1 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/DataSourceGroupType.java @@ -21,9 +21,9 @@ * Entities groups */ public enum DataSourceGroupType { - Marketing("Marketing Campaign Objects"), - Stats("Statistic Objects"), - Suppressions("Suppression Objects"); + Marketing("MarketingCampaign"), + Stats("Statistic"), + Suppressions("suppression"); private String value; diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java index 48a78a4..be44d3c 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/AdvancedStats.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; @@ -37,7 +37,7 @@ Group = DataSourceGroupType.Stats, APIUrl = "geo/stats", RequiredArguments = { - BaseSourceConfig.PROPERTY_START_DATE + BaseConfig.PROPERTY_START_DATE } ) public class AdvancedStats extends BaseObject implements IBaseObject { @@ -46,7 +46,7 @@ public class AdvancedStats extends BaseObject implements IBaseObject { @SerializedName("date") private String date; - @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY) + @ObjectFieldDefinition(FieldType = Schema.Type.ARRAY, NestedClass = "StatsStats") @SerializedName("stats") private List stats; diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java index b95aee7..49d2c3b 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/CategoryStats.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; @@ -37,8 +37,8 @@ Group = DataSourceGroupType.Stats, APIUrl = "categories/stats", RequiredArguments = { - BaseSourceConfig.PROPERTY_START_DATE, - BaseSourceConfig.PROPERTY_STAT_CATEGORIES + BaseConfig.PROPERTY_START_DATE, + BaseConfig.PROPERTY_STAT_CATEGORIES } ) public class CategoryStats extends BaseObject implements IBaseObject { diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java index 8d30c11..879399e 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/GlobalStats.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; @@ -37,7 +37,7 @@ Group = DataSourceGroupType.Stats, APIUrl = "stats", RequiredArguments = { - BaseSourceConfig.PROPERTY_START_DATE + BaseConfig.PROPERTY_START_DATE } ) public class GlobalStats extends BaseObject implements IBaseObject { diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java index caf7510..390d91a 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/MetricStats.java @@ -17,9 +17,11 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; import java.util.Map; @@ -34,62 +36,77 @@ ) public class MetricStats extends BaseObject implements IBaseObject { + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("blocks") private Integer blocks; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("bounce_drops") private Integer bounceDrops; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("bounces") private Integer bounces; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("clicks") private Integer clicks; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("deferred") private Integer deferred; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("invalid_emails") private Integer invalidEmails; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("opens") private Integer opens; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("processed") private Integer processed; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("requests") private Integer requests; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("spam_report_drops") private Integer spamReportDrops; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("spam_reports") private Integer spamReports; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("unique_clicks") private Integer uniqueClicks; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("unique_opens") private Integer uniqueOpens; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("unsubscribe_drops") private Integer unsubscribeDrops; + @ObjectFieldDefinition(FieldType = Schema.Type.INT) @Nullable @SerializedName("unsubscribes") private Integer unsubscribes; diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java index 8c7ace4..300fd79 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/stats/StatsStats.java @@ -17,9 +17,11 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.sendgrid.common.helpers.BaseObject; import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; +import io.cdap.plugin.sendgrid.common.helpers.ObjectFieldDefinition; import java.util.Map; @@ -32,12 +34,15 @@ ) public class StatsStats extends BaseObject implements IBaseObject { + @ObjectFieldDefinition(FieldType = Schema.Type.MAP, NestedClass = "MetricStats") @SerializedName("metrics") private MetricStats metrics; + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) @SerializedName("name") private String name; + @ObjectFieldDefinition(FieldType = Schema.Type.STRING) @SerializedName("type") private String type; diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSourceConfig.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchConfig.java similarity index 67% rename from src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSourceConfig.java rename to src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchConfig.java index 2663a8b..eef9566 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchConfig.java @@ -15,13 +15,18 @@ */ package io.cdap.plugin.sendgrid.source.batch; -import io.cdap.plugin.sendgrid.common.config.BaseSourceConfig; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; /** * SendGrid Source Plugin configuration */ -public class SendGridBatchSourceConfig extends BaseSourceConfig { - public SendGridBatchSourceConfig(String referenceName) { +public class SendGridBatchConfig extends BaseConfig { + public SendGridBatchConfig(String referenceName) { super(referenceName); } + + public void validate(FailureCollector failureCollector) { + new SendGridBatchConfigValidator(failureCollector, this).validate(); + } } diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchConfigValidator.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchConfigValidator.java new file mode 100644 index 0000000..9564467 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchConfigValidator.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.source.batch; + +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.common.config.BaseConfigValidator; + +/** + * Validates configuration + */ +public class SendGridBatchConfigValidator extends BaseConfigValidator { + public SendGridBatchConfigValidator(FailureCollector failureCollector, SendGridBatchConfig config) { + super(failureCollector, config); + } + + @Override + public void validate() { + super.validate(); + } + +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java index 11c82b3..7949eee 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridBatchSource.java @@ -46,9 +46,9 @@ public class SendGridBatchSource extends BatchSource { public static final String NAME = "SendGrid"; - private final SendGridBatchSourceConfig config; + private final SendGridBatchConfig config; - public SendGridBatchSource(SendGridBatchSourceConfig config) { + public SendGridBatchSource(SendGridBatchConfig config) { this.config = config; } diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java index f5ac25f..6a6a6fc 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormat.java @@ -45,7 +45,7 @@ public RecordReader createRecordReader(InputSplit split, TaskAttemptContext cont Configuration conf = context.getConfiguration(); String serializedConfig = conf.get(SendGridInputFormatProvider.PROPERTY_CONFIG_JSON); - SendGridBatchSourceConfig sgConfig = gson.fromJson(serializedConfig, SendGridBatchSourceConfig.class); + SendGridBatchConfig sgConfig = gson.fromJson(serializedConfig, SendGridBatchConfig.class); return (sgConfig.isMultiObjectMode()) ? new SendGridMultiRecordReader() diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java index 124c07c..ac37907 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridInputFormatProvider.java @@ -30,7 +30,7 @@ public class SendGridInputFormatProvider implements InputFormatProvider { private static final Gson gson = new GsonBuilder().create(); private final Map conf; - SendGridInputFormatProvider(SendGridBatchSourceConfig config) { + SendGridInputFormatProvider(SendGridBatchConfig config) { this.conf = new ImmutableMap.Builder() .put(PROPERTY_CONFIG_JSON, gson.toJson(config)) .build(); diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java index 0d0b009..5ecdbab 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridMultiRecordReader.java @@ -46,7 +46,7 @@ public class SendGridMultiRecordReader extends RecordReader> iterators = new ImmutableMap.Builder<>(); diff --git a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java index 39955e8..1b980e4 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java +++ b/src/main/java/io/cdap/plugin/sendgrid/source/batch/SendGridRecordReader.java @@ -44,7 +44,7 @@ public class SendGridRecordReader extends RecordReader { + StructuredRecord.Builder itemBuilder = StructuredRecord.builder(Objects.requireNonNull(componentSchema)); + transformValue(k, arrItem, componentSchema, itemBuilder); + return builder.build(); + }).collect(Collectors.toList()); + builder.set(k, values); } else { builder.set(k, v); } diff --git a/widgets/SendGrid-batchsource.json b/widgets/SendGrid-batchsource.json index e45e82c..2755c6f 100644 --- a/widgets/SendGrid-batchsource.json +++ b/widgets/SendGrid-batchsource.json @@ -55,15 +55,15 @@ "widget-attributes": { "options": [ { - "id": "campaigns", + "id": "MarketingCampaign", "label": "Marketing Campaign Objects" }, { - "id": "stats", + "id": "Statistic", "label": "Statistic Objects" }, { - "id": "suppressions", + "id": "suppression", "label": "Suppression Objects" } ], @@ -116,23 +116,7 @@ }, { "id": "GlobalStats", - "label": "Advanced Stats" - }, - { - "id": "_suppressions", - "label": "Suppression Objects" - }, - { - "id": "Bounces", - "label": "Bounces" - }, - { - "id": "GlobalUnsubscribes", - "label": "Global Unsubscribes" - }, - { - "id": "GroupUnsubscribes", - "label": "Group Unsubscribes" + "label": "Global Stats" } ], "delimiter": "," @@ -312,13 +296,13 @@ } }, { - "name": "startDate", + "name": "start_date", "label": "Start Date", "widget-type": "textbox", "placeholder": "YYYY-MM-DD" }, { - "name": "endDate", + "name": "end_date", "label": "End Date", "widget-type": "textbox", "placeholder": "YYYY-MM-DD" From 746d1b0109acc4cd2dea164eb2f27d34adff4a99 Mon Sep 17 00:00:00 2001 From: Dmitry Grinenko Date: Fri, 1 Nov 2019 10:44:52 +0200 Subject: [PATCH 4/6] Add SendGrid Sink Plugin --- checkstyle.xml | 4 +- docs/SendGrid-batchsink.md | 0 icons/SendGrid-batchsink.png | Bin 0 -> 442 bytes pom.xml | 11 + .../batch/sink/SendGridOutputFormat.java | 72 +++++ .../sink/SendGridOutputFormatProvider.java | 48 +++ .../batch/sink/SendGridRecordWriter.java | 54 ++++ .../sendgrid/batch/sink/SendGridSink.java | 89 ++++++ .../batch/sink/SendGridSinkConfig.java | 227 ++++++++++++++ .../sink/SendGridSinkConfigValidator.java | 83 ++++++ .../batch/sink/SendGridSinkTransformer.java | 79 +++++ .../source}/SendGridInputFormat.java | 10 +- .../source}/SendGridInputFormatProvider.java | 4 +- .../source}/SendGridMultiRecordReader.java | 16 +- .../source}/SendGridRecordReader.java | 16 +- .../source/SendGridSource.java} | 17 +- .../batch/source/SendGridSourceConfig.java | 217 ++++++++++++++ .../source/SendGridSourceConfigValidator.java | 167 +++++++++++ .../source/SendGridSourceTransformer.java} | 13 +- .../batch => batch/source}/SendGridSplit.java | 11 +- .../sendgrid/common/SendGridClient.java | 31 +- .../sendgrid/common/config/BaseConfig.java | 185 +----------- .../common/config/BaseConfigValidator.java | 145 ++------- .../sendgrid/common/helpers/BaseObject.java | 7 +- .../common/helpers/ObjectDefinition.java | 9 +- .../sendgrid/common/helpers/ObjectHelper.java | 24 +- .../common/objects/DataSourceGroupType.java | 3 +- .../common/objects/mail/SendGridMail.java | 144 +++++++++ .../objects/mail/SendGridMailBuilder.java | 184 ++++++++++++ .../objects/mail/SendGridMailContent.java} | 28 +- .../objects/mail/SendGridMailFooter.java | 62 ++++ .../objects/mail/SendGridMailPerson.java} | 20 +- .../objects/mail/SendGridMailSettings.java | 50 ++++ .../mail/SendGridPersonalizations.java | 68 +++++ .../common/objects/mail/SendGridSwitch.java | 39 +++ .../mail/SendGridTrackingSettings.java | 51 ++++ .../common/objects/stats/AdvancedStats.java | 4 +- .../common/objects/stats/CategoryStats.java | 6 +- .../common/objects/stats/GlobalStats.java | 4 +- suppressions.xml | 4 +- widgets/SendGrid-batchsink.json | 282 ++++++++++++++++++ widgets/SendGrid-batchsource.json | 50 +--- 42 files changed, 2094 insertions(+), 444 deletions(-) create mode 100644 docs/SendGrid-batchsink.md create mode 100644 icons/SendGrid-batchsink.png create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormat.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridOutputFormatProvider.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridRecordWriter.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSink.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigValidator.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkTransformer.java rename src/main/java/io/cdap/plugin/sendgrid/{source/batch => batch/source}/SendGridInputFormat.java (82%) rename src/main/java/io/cdap/plugin/sendgrid/{source/batch => batch/source}/SendGridInputFormatProvider.java (93%) rename src/main/java/io/cdap/plugin/sendgrid/{source/batch => batch/source}/SendGridMultiRecordReader.java (84%) rename src/main/java/io/cdap/plugin/sendgrid/{source/batch => batch/source}/SendGridRecordReader.java (81%) rename src/main/java/io/cdap/plugin/sendgrid/{source/batch/SendGridBatchSource.java => batch/source/SendGridSource.java} (88%) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigValidator.java rename src/main/java/io/cdap/plugin/sendgrid/{source/batch/SendGridTransformer.java => batch/source/SendGridSourceTransformer.java} (86%) rename src/main/java/io/cdap/plugin/sendgrid/{source/batch => batch/source}/SendGridSplit.java (73%) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMail.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailBuilder.java rename src/main/java/io/cdap/plugin/sendgrid/{source/batch/SendGridBatchConfigValidator.java => common/objects/mail/SendGridMailContent.java} (57%) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailFooter.java rename src/main/java/io/cdap/plugin/sendgrid/{source/batch/SendGridBatchConfig.java => common/objects/mail/SendGridMailPerson.java} (57%) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridMailSettings.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridPersonalizations.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridSwitch.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/mail/SendGridTrackingSettings.java create mode 100644 widgets/SendGrid-batchsink.json diff --git a/checkstyle.xml b/checkstyle.xml index c77494c..f9a301a 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -13,8 +13,8 @@ --> + "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" + "https://checkstyle.org/dtds/configuration_1_3.dtd"> + "-//Checkstyle//DTD SuppressionFilter Configuration 1.1//EN" + "https://checkstyle.org/dtds/suppressions_1_1.dtd"> diff --git a/widgets/SendGrid-batchsink.json b/widgets/SendGrid-batchsink.json new file mode 100644 index 0000000..47ed0cd --- /dev/null +++ b/widgets/SendGrid-batchsink.json @@ -0,0 +1,282 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "name": "referenceName", + "label": "Reference Name", + "widget-type": "textbox" + }, + { + "name": "authType", + "label": "Authentication type", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "basic", + "options": [ + { + "id": "basic", + "label": "Basic" + }, + { + "id": "api", + "label": "API Key" + } + ] + } + }, + { + "name": "sendGridApiKey", + "label": "API Key", + "widget-type": "securekey-text", + "widget-attributes": { + "placeholder": "SendGrid API Key" + } + }, + { + "name": "username", + "label": "Username", + "widget-type": "textbox" + }, + { + "name": "password", + "label": "Password", + "widget-type": "password" + }, + { + "name": "from", + "label": "From", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "some@email.com" + } + }, + { + "name": "recipientAddressSource", + "label": "Recipient address source", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "input", + "options": [ + { + "id": "input", + "label": "Input Record" + }, + { + "id": "config", + "label": "Configuration" + } + ] + } + }, + { + "name": "recipientColumnName", + "label": "Column name for recipients addresses", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "recipients column name" + } + }, + { + "name": "recipientConfigAddressList", + "label": "Recipient address list", + "widget-type": "csv", + "widget-attributes": { + "delimiter": ",", + "value-placeholder": "some@mail.com" + } + }, + { + "name": "mailSubject", + "label": "Email subject", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "email subject" + } + }, + { + "name": "bodyColumnName", + "label": "Email body column name", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "body column name" + } + }, + { + "name": "replyTo", + "label": "Reply To", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "some@email.com" + } + } + ] + }, + { + "label": "Options", + "properties": [ + { + "name": "footerEnabled", + "label": "Custom Mail Footer", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "footerHtml", + "label": "Footer HTML", + "widget-type": "textarea", + "widget-attributes": { + "rows": 5, + "placeholder": "footer message" + } + }, + { + "name": "sandboxMode", + "label": "Sandbox Mode", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "clickTracking", + "label": "Click Tracking", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "openTracking", + "label": "Open Tracking", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + }, + { + "name": "subscriptionTracking", + "label": "Subscription Tracking", + "widget-type": "toggle", + "widget-attributes": { + "on": { + "value": "true", + "label": "On" + }, + "off": { + "value": "false", + "label": "Off" + }, + "default": "false" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "schema-types": [ + "string" + ], + "schema-default-type": "string" + } + } + ], + "filters": [ + { + "name": "AuthTypeBasicFilter", + "condition": { + "expression": "authType == 'basic'" + }, + "show": [ + { + "name": "username", + "type": "property" + }, + { + "name": "password", + "type": "property" + } + ] + }, + { + "name": "AuthTypeAPIKeyFilter", + "condition": { + "expression": "authType == 'api'" + }, + "show": [ + { + "name": "sendGridApiKey", + "type": "property" + } + ] + }, + { + "name": "recipientSourceFilter", + "condition": { + "expression": "recipientAddressSource == 'input'" + }, + "show": [ + { + "name": "recipientColumnName", + "type": "property" + } + ] + }, + { + "name": "recipientSourceFilterConfig", + "condition": { + "expression": "recipientAddressSource == 'config'" + }, + "show": [ + { + "name": "recipientConfigAddressList", + "type": "property" + } + ] + } + ] +} \ No newline at end of file diff --git a/widgets/SendGrid-batchsource.json b/widgets/SendGrid-batchsource.json index 2755c6f..37809ec 100644 --- a/widgets/SendGrid-batchsource.json +++ b/widgets/SendGrid-batchsource.json @@ -234,6 +234,10 @@ "id" : "message_count", "label": "message_count" }, + { + "id": "metrics", + "label": "metrics" + }, { "id" : "name", "label": "name" @@ -291,10 +295,14 @@ "label": "zip" } ], - "delimiter": ",", - "defualt": "" + "delimiter": "," } - }, + } + ] + }, + { + "label": "Options", + "properties": [ { "name": "start_date", "label": "Start Date", @@ -342,45 +350,15 @@ }, { "name": "AuthTypeAPIKeyFilter", - "condition": "authType == 'api'", + "condition": { + "expression": "authType == 'api'" + }, "show": [ { "name": "sendGridApiKey", "type": "property" } ] - }, - { - "name": "MarketingFilter", - "condition": "dataSourceTypes.includes('campaigns')", - "show": [ - { - "name": "dataSourceMarketing", - "type": "property" - } - ] - }, - { - "name": "StatsFilter", - "condition": "dataSourceTypes.includes('stats')", - "show": [ - { - "name": "dataSourceStats", - "type": "property" - } - ] - }, - { - "name": "SuppressionsFilter", - "condition": "dataSourceTypes.includes('suppressions')", - "show": [ - { - "name": "dataSourceSuppressions", - "type": "property" - } - ] } - ] - } \ No newline at end of file From 2cdcdfa4edde0a6d04c7d2f638ce169386c0be01 Mon Sep 17 00:00:00 2001 From: Dmitry Grinenko Date: Fri, 8 Nov 2019 14:31:11 +0200 Subject: [PATCH 5/6] Adding integration test --- pom.xml | 56 ++++++ .../batch/sink/SendGridSinkConfig.java | 28 ++- .../batch/source/SendGridSourceConfig.java | 2 +- .../sendgrid/common/SendGridClient.java | 32 +++- .../sendgrid/common/config/BaseConfig.java | 2 +- .../sendgrid/common/helpers/ObjectHelper.java | 50 ++--- .../marketing/MarketingNewContact.java | 120 ++++++++++++ .../marketing/MarketingNewContacts.java | 57 ++++++ .../io/cdap/plugin/sendgrid/BaseTest.java | 57 ++++++ .../batch/sink/SendGridSinkConfigTest.java | 104 ++++++++++ .../source/SendGridSourceConfigTest.java | 77 ++++++++ .../common/config/BaseConfigTest.java | 111 +++++++++++ .../sendgrid/etl/SendGridSourceTest.java | 181 ++++++++++++++++++ src/test/resources/BaseConfigExample.json | 6 + src/test/resources/KeyConfigExample.json | 6 + .../resources/SendGridSinkConfigExample.json | 15 ++ .../SendGridSourceConfigExample.json | 10 + src/test/resources/new_contacts.csv | 4 + 18 files changed, 889 insertions(+), 29 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContact.java create mode 100644 src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContacts.java create mode 100644 src/test/java/io/cdap/plugin/sendgrid/BaseTest.java create mode 100644 src/test/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigTest.java create mode 100644 src/test/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigTest.java create mode 100644 src/test/java/io/cdap/plugin/sendgrid/common/config/BaseConfigTest.java create mode 100644 src/test/java/io/cdap/plugin/sendgrid/etl/SendGridSourceTest.java create mode 100644 src/test/resources/BaseConfigExample.json create mode 100644 src/test/resources/KeyConfigExample.json create mode 100644 src/test/resources/SendGridSinkConfigExample.json create mode 100644 src/test/resources/SendGridSourceConfigExample.json create mode 100644 src/test/resources/new_contacts.csv diff --git a/pom.xml b/pom.xml index 52ef85d..7d94cab 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,9 @@ 2.3.0-SNAPSHOT 2.2.4 1.2 + 4.11 + 1.10.19 + 2.1.3 @@ -249,6 +252,36 @@ + + io.cdap.cdap + hydrator-test + ${cdap.version} + test + + + junit + junit + ${junit.version} + test + + + io.cdap.cdap + cdap-data-pipeline + ${cdap.version} + test + + + org.mockito + mockito-all + ${mockito.version} + test + + + com.google.inject + guice + 4.2.2 + test + @@ -332,6 +365,29 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 2.14.1 + + -Xmx5000m -Djava.awt.headless=true -XX:+UseG1GC -XX:OnOutOfMemoryError="kill -9 %p" -Djava.net.preferIPv4Stack=true + false + plain + + ${project.build.directory} + + + **/*TestsSuite.java + **/*TestSuite.java + **/Test*.java + **/*Test.java + **/*TestCase.java + + + **/*TestRun.java + + + \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java index 60a407c..126349b 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java @@ -164,8 +164,32 @@ public void validate(FailureCollector failureCollector) { } public void validate(Schema schema) { - //ToDo 10/29/2019/hapyl: implement this - assert schema != null; + if (schema == null) { + throw new IllegalArgumentException("Input schema cannot be empty"); + } + + if (getRecipientAddressSource() == ToAddressSource.INPUT) { + Schema.Field recipient = schema.getField(recipientColumnName); + if (recipient == null) { + throw new IllegalArgumentException(String.format("Plugin is configured to use column '%s' for" + + " recipient addresses, but input schema did not provide such column", recipientColumnName)); + } + + if (recipient.getSchema().getType() != Schema.Type.STRING) { + throw new IllegalArgumentException(String.format("The input schema column '%s' expected to be of type STRING", + recipientColumnName)); + } + } + + Schema.Field body = schema.getField(bodyColumnName); + if (body == null) { + throw new IllegalArgumentException(String.format("Plugin requires column '%s' for" + + " recipient addresses, but input schema did not provide such column", recipientColumnName)); + } + if (body.getSchema().getType() != Schema.Type.STRING) { + throw new IllegalArgumentException(String.format("The input schema column '%s' expected to be of type STRING", + bodyColumnName)); + } } public String getFrom() { diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java index 5c3e42d..2240124 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfig.java @@ -193,7 +193,7 @@ public Map getRequestArguments() { builder.put(PROPERTY_END_DATE, endDate); } if (!Strings.isNullOrEmpty(statCategories)) { - builder.put(PROPERTY_END_DATE, statCategories); + builder.put(PROPERTY_STAT_CATEGORIES, statCategories); } return builder.build(); } diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java index c5db1cf..a6c07ce 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java @@ -16,6 +16,7 @@ package io.cdap.plugin.sendgrid.common; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; @@ -30,6 +31,7 @@ import io.cdap.plugin.sendgrid.common.objects.BasicResult; import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; import io.cdap.plugin.sendgrid.common.objects.mail.SendGridMail; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingNewContacts; import java.io.IOException; import java.lang.reflect.ParameterizedType; @@ -121,7 +123,7 @@ private String makeApiRequest(Method method, String endpoint, @Nullable Map ids) throws IOException { + ObjectInfo objectInfo = ObjectHelper.getObjectInfoFromClass(MarketingNewContacts.class); + String idsToRemove = String.join(",", ids); + Map args = new ImmutableMap.Builder() + .put("delete_all_contacts", "false") + .put("ids", idsToRemove) + .build(); + makeApiRequest(Method.DELETE, objectInfo.getSendGridAPIUrl(), args, null); + } + /** * Query SendGrid API using plugin meta objects * diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java index 63ff782..5c111fb 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/config/BaseConfig.java @@ -40,7 +40,7 @@ public abstract class BaseConfig extends ReferencePluginConfig { @Macro private String authType; - @Name((PROPERTY_SENDGRID_API_KEY)) + @Name(PROPERTY_SENDGRID_API_KEY) @Description("The SendGrid API Key taken from the SendGrid account") @Macro @Nullable diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java index c9b1648..2b0d870 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java +++ b/src/main/java/io/cdap/plugin/sendgrid/common/helpers/ObjectHelper.java @@ -90,6 +90,31 @@ public class ObjectHelper { buildSchemaDefinition(); } + public static ObjectInfo getObjectInfoFromClass(Class object) { + ObjectDefinition objectDefinition = (ObjectDefinition) object.getAnnotation(ObjectDefinition.class); + + List objectFieldInfos = Arrays.stream(object.getDeclaredFields()).map(x -> { + try { + String name = x.getAnnotation(SerializedName.class).value(); + ObjectFieldDefinition objectFieldDefinition = x.getAnnotation(ObjectFieldDefinition.class); + + return new ObjectFieldInfo(name, objectFieldDefinition.FieldType(), objectFieldDefinition.NestedClass()); + } catch (NullPointerException e) { + return null; // Ignore non-annotated fields + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + return new ObjectInfo( + objectDefinition.Name(), + objectFieldInfos, + objectDefinition.APIUrl(), + objectDefinition.APIResponseType(), + object, + objectDefinition.Group(), + Arrays.asList(objectDefinition.RequiredArguments()), + objectDefinition.ObjectType() + ); + } + /** * Create schema definition for annotated objects */ @@ -100,30 +125,7 @@ private static void buildSchemaDefinition() { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); objects.forEach(object -> { try { - ObjectDefinition objectDefinition = (ObjectDefinition) object.getAnnotation(ObjectDefinition.class); - - List objectFieldInfos = Arrays.stream(object.getDeclaredFields()).map(x -> { - try { - String name = x.getAnnotation(SerializedName.class).value(); - ObjectFieldDefinition objectFieldDefinition = x.getAnnotation(ObjectFieldDefinition.class); - - return new ObjectFieldInfo(name, objectFieldDefinition.FieldType(), objectFieldDefinition.NestedClass()); - } catch (NullPointerException e) { - return null; // Ignore non-annotated fields - } - }).filter(Objects::nonNull).collect(Collectors.toList()); - - builder.put(object.getName(), new ObjectInfo( - objectDefinition.Name(), - objectFieldInfos, - objectDefinition.APIUrl(), - objectDefinition.APIResponseType(), - object, - objectDefinition.Group(), - Arrays.asList(objectDefinition.RequiredArguments()), - objectDefinition.ObjectType() - )); - + builder.put(object.getName(), getObjectInfoFromClass(object)); } catch (NullPointerException e) { throw new RuntimeException(String.format("Object with name %s not annotated with %s", object.getName(), ObjectDefinition.class.getName())); diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContact.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContact.java new file mode 100644 index 0000000..1c4df04 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContact.java @@ -0,0 +1,120 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.gson.annotations.SerializedName; + +/** + * Object for creating new SendGrid contacts + */ +public class MarketingNewContact { + @SerializedName("address_line_1") + private String addressLine1; + + @SerializedName("address_line_2") + private String addressLine2; + + @SerializedName("city") + private String city; + + @SerializedName("country") + private String country; + + @SerializedName("email") + private String email; + + @SerializedName("first_name") + private String firstName; + + @SerializedName("last_name") + private String lastName; + + @SerializedName("postal_code") + private String postalCode; + + @SerializedName("state_province_region") + private String stateProvinceRegion; + + /** + * Initializes class from csv line. + * + * Column format: + * email,first_name,last_name,address_line_1,address_line_2,city,state_province_region,postal_code,country + */ + public MarketingNewContact(String csvLine) { + String[] columns = csvLine.split(","); + if (columns.length != 9) { + throw new IllegalArgumentException(String.format("Invalid csv formatted line: '%s'", csvLine)); + } + this.email = columns[0]; + this.firstName = columns[1]; + this.lastName = columns[2]; + this.addressLine1 = columns[3]; + this.addressLine2 = columns[4]; + this.city = columns[5]; + this.stateProvinceRegion = columns[6]; + this.postalCode = columns[7]; + this.country = columns[8]; + } + + public static MarketingNewContact fromCSVLine(String csv) { + return new MarketingNewContact(csv); + } + + public String getAddressLine1() { + return addressLine1; + } + + public String getCity() { + return city; + } + + public String getCountry() { + return country; + } + + public String getEmail() { + return email; + } + + public String getFirstName() { + return firstName; + } + + public String getLastName() { + return lastName; + } + + public String getPostalCode() { + return postalCode; + } + + public String getStateProvinceRegion() { + return stateProvinceRegion; + } + + public String getAddressLine2() { + return addressLine2; + } + + public void setLastName(String value) { + this.lastName = value; + } + + public void setEmail(String value) { + this.email = value; + } +} diff --git a/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContacts.java b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContacts.java new file mode 100644 index 0000000..fedd3f1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sendgrid/common/objects/marketing/MarketingNewContacts.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.objects.marketing; + +import com.google.gson.annotations.SerializedName; +import io.cdap.plugin.sendgrid.common.helpers.BaseObject; +import io.cdap.plugin.sendgrid.common.helpers.IBaseObject; +import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Object for creating new SendGrid contacts + */ +@ObjectDefinition(APIUrl = "marketing/contacts") +public class MarketingNewContacts extends BaseObject implements IBaseObject { + @SerializedName("list_ids") + private List listIds; + + @SerializedName("contacts") + private List contacts; + + public MarketingNewContacts(List csvLines) { + listIds = new ArrayList<>(); + contacts = csvLines.stream().map(MarketingNewContact::fromCSVLine).collect(Collectors.toList()); + } + + public MarketingNewContacts(String csvLines) { + this(Arrays.asList(csvLines.split(System.lineSeparator()))); + } + + @Override + public Map asMap() { + return null; + } + + public List getContacts() { + return contacts; + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/BaseTest.java b/src/test/java/io/cdap/plugin/sendgrid/BaseTest.java new file mode 100644 index 0000000..156716d --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/BaseTest.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * Base test class for SendGrid plugin + */ +public class BaseTest { + + /** + * Read resource as {@link String} + * + * @param name resource name + * @return resource string representation + * @throws IOException in case if resource not found + */ + public static String getResource(String name) throws IOException { + ClassLoader classLoader = BaseTest.class.getClassLoader(); + + try (InputStream inputStream = classLoader.getResourceAsStream(name)) { + if (inputStream == null) { + throw new IOException(String.format("Error in reading file '%s'", name)); + } + try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream)) { + BufferedReader buffer = new BufferedReader(inputStreamReader); + return buffer.lines().collect(Collectors.joining(System.lineSeparator())); + } + } + } + + /** + * Returns random uuid without "-" + */ + public static String getRandomUUID() { + return UUID.randomUUID().toString().replace("-", ""); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigTest.java b/src/test/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigTest.java new file mode 100644 index 0000000..8f7d1aa --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfigTest.java @@ -0,0 +1,104 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.sink; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.BaseTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + + +public class SendGridSinkConfigTest extends BaseTest { + private static Gson gson = new GsonBuilder().create(); + SendGridSinkConfig config; + + @Before + public void setUp() throws Exception { + config = gson.fromJson(getResource("SendGridSinkConfigExample.json"), SendGridSinkConfig.class); + } + + @Test + public void testGetMailSubject() { + Assert.assertEquals("subject", config.getMailSubject()); + } + + @Test + public void testGetFrom() { + Assert.assertEquals("test@email.com", config.getFrom()); + } + + @Test + public void testGetRecipientAddressSource() { + Assert.assertEquals(SendGridSinkConfig.ToAddressSource.INPUT, config.getRecipientAddressSource()); + } + + @Test + public void testGetRecipientAddresses() { + Assert.assertEquals( + Arrays.asList("test1@email.com", "test2@email.com"), + config.getRecipientAddresses() + ); + } + + @Test + public void testGetRecipientColumnName() { + Assert.assertEquals("column1", config.getRecipientColumnName()); + } + + @Test + public void testGetBodyColumnName() { + Assert.assertEquals("column2", config.getBodyColumnName()); + } + + @Test + public void testGetReplyTo() { + Assert.assertEquals("reply@email.com", config.getReplyTo()); + } + + @Test + public void testGetFooterEnable() { + Assert.assertEquals(true, config.getFooterEnable()); + } + + @Test + public void testGetFooterHTML() { + Assert.assertEquals("footer", config.getFooterHTML()); + } + + @Test + public void testGetSandboxMode() { + Assert.assertEquals(true, config.getSandboxMode()); + } + + @Test + public void testGetClickTracking() { + Assert.assertEquals(true, config.getClickTracking()); + } + + @Test + public void testGetOpenTracking() { + Assert.assertEquals(true, config.getOpenTracking()); + } + + @Test + public void testGetSubscriptionTracking() { + Assert.assertEquals(true, config.getSubscriptionTracking()); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigTest.java b/src/test/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigTest.java new file mode 100644 index 0000000..dee4d58 --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/batch/source/SendGridSourceConfigTest.java @@ -0,0 +1,77 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.batch.source; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.sendgrid.BaseTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +public class SendGridSourceConfigTest extends BaseTest { + private static Gson gson = new GsonBuilder().create(); + SendGridSourceConfig config; + + @Before + public void setUp() throws Exception { + config = gson.fromJson(getResource("SendGridSourceConfigExample.json"), SendGridSourceConfig.class); + } + + @Test + public void getFields() { + Assert.assertEquals( + Arrays.asList("address", "city", "contracts_count"), + config.getFields() + ); + } + + @Test + public void getDataSource() { + Assert.assertEquals( + Arrays.asList("SingleSends", "Senders", "CategoryStats", "Bounces", "GlobalUnsubscribes"), + config.getDataSource() + ); + } + + @Test + public void isMultiObjectMode() { + Assert.assertTrue(config.isMultiObjectMode()); + } + + @Test + public void getRequestArguments() { + Assert.assertEquals( + new ImmutableMap.Builder() + .put(SendGridSourceConfig.PROPERTY_START_DATE, "2019-09-18") + .put(SendGridSourceConfig.PROPERTY_END_DATE, "2019-09-21") + .put(SendGridSourceConfig.PROPERTY_STAT_CATEGORIES, "spam") + .build(), + config.getRequestArguments() + ); + } + + @Test + public void getDataSourceTypes() { + Assert.assertEquals( + Arrays.asList("MarketingCampaign", "Statistic", "Suppression"), + config.getDataSourceTypes() + ); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/common/config/BaseConfigTest.java b/src/test/java/io/cdap/plugin/sendgrid/common/config/BaseConfigTest.java new file mode 100644 index 0000000..90c4833 --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/common/config/BaseConfigTest.java @@ -0,0 +1,111 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.common.config; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.InstanceCreator; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sendgrid.BaseTest; +import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.lang.reflect.Type; + + +public class BaseConfigTest extends BaseTest { + /** + * Custom class creator for abstract {@link BaseConfig} + */ + private static class BaseConfigInstanceCreator implements InstanceCreator { + + private String ref; + + BaseConfigInstanceCreator(String ref) { + this.ref = ref; + } + + @Override + public BaseConfig createInstance(Type type) { + return new BaseConfig(ref) { + @Override + protected void validate(FailureCollector failureCollector) { + // no-op + throw new IllegalArgumentException("no op method"); + } + }; + } + } + + private BaseConfig basicAuthConfig; + private BaseConfig keyAuthConfig; + + private static Gson gson; + private static String refName = "ref" + getRandomUUID(); + + @BeforeClass + public static void classSetUp() { + GsonBuilder builder = new GsonBuilder(); + builder.registerTypeAdapter(BaseConfig.class, new BaseConfigInstanceCreator(refName)); + gson = builder.create(); + } + + @Before + public void setUp() throws Exception { + basicAuthConfig = gson.fromJson(getResource("BaseConfigExample.json"), BaseConfig.class); + keyAuthConfig = gson.fromJson(getResource("KeyConfigExample.json"), BaseConfig.class); + } + + @Test + public void testReferenceValue() { + Assert.assertEquals(refName, basicAuthConfig.referenceName); + Assert.assertEquals(refName, keyAuthConfig.referenceName); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateBasic() { + basicAuthConfig.validate(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateKey() { + keyAuthConfig.validate(null); + } + + @Test + public void testGetAuthType() { + Assert.assertEquals(SendGridAuthType.BASIC, basicAuthConfig.getAuthType()); + Assert.assertEquals(SendGridAuthType.API, keyAuthConfig.getAuthType()); + } + + @Test + public void testGetSendGridApiKey() { + Assert.assertEquals("some-api-key", keyAuthConfig.getSendGridApiKey()); + } + + @Test + public void testGetAuthUserName() { + Assert.assertEquals("test user", basicAuthConfig.getAuthUserName()); + } + + @Test + public void testGetAuthPassword() { + Assert.assertEquals("test pass", basicAuthConfig.getAuthPassword()); + } +} diff --git a/src/test/java/io/cdap/plugin/sendgrid/etl/SendGridSourceTest.java b/src/test/java/io/cdap/plugin/sendgrid/etl/SendGridSourceTest.java new file mode 100644 index 0000000..93c15b7 --- /dev/null +++ b/src/test/java/io/cdap/plugin/sendgrid/etl/SendGridSourceTest.java @@ -0,0 +1,181 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sendgrid.etl; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.datapipeline.DataPipelineApp; +import io.cdap.cdap.datapipeline.SmartWorkflow; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.mock.batch.MockSink; +import io.cdap.cdap.etl.mock.test.HydratorTestBase; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.ProgramRunStatus; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.DataSetManager; +import io.cdap.cdap.test.WorkflowManager; +import io.cdap.plugin.sendgrid.BaseTest; +import io.cdap.plugin.sendgrid.batch.source.SendGridSource; +import io.cdap.plugin.sendgrid.batch.source.SendGridSourceConfig; +import io.cdap.plugin.sendgrid.common.SendGridClient; +import io.cdap.plugin.sendgrid.common.config.BaseConfig; +import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingContacts; +import io.cdap.plugin.sendgrid.common.objects.marketing.MarketingNewContacts; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class SendGridSourceTest extends HydratorTestBase { + private static final ArtifactSummary APP_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); + private static final String LAST_NAME_FIELD = "last_name"; + private static final String ID_FIELD = "id"; + private static final String BASIC_AUTH_TYPE = "basic"; + private static final String REPLACE_PATTERN = "%id%"; + + private static String authType; + private static String authUser; + private static String authPass; + private static String authToken; + + private static SendGridClient client; + private static MarketingNewContacts contacts; + private static String reference; + private static int createdContactsAmount; + + @BeforeClass + public static void setupTestClass() throws Exception { + reference = BaseTest.getRandomUUID(); + authType = System.getProperty("sg.auth.type"); + if (authType != null && authType.equals(BASIC_AUTH_TYPE)) { + authUser = System.getProperty("sg.auth.user"); + authPass = System.getProperty("sg.auth.pass"); + + if (authUser == null || authPass == null) { + throw new IllegalArgumentException("'sg.auth.user' and 'sg.auth.pass' system property must not be empty"); + } + client = new SendGridClient(authUser, authPass); + } else { + authToken = System.getProperty("sg.auth.token"); + if (authToken == null) { + throw new IllegalArgumentException("'sg.auth.token' system property must not be empty"); + } + client = new SendGridClient(authToken); + } + + + ArtifactId parentArtifact = NamespaceId.DEFAULT.artifact(APP_ARTIFACT.getName(), APP_ARTIFACT.getVersion()); + setupBatchArtifacts(parentArtifact, DataPipelineApp.class); + addPluginArtifact( + NamespaceId.DEFAULT.artifact("example-plugins", "1.0.0"), + parentArtifact, + SendGridSource.class + ); + contacts = new MarketingNewContacts(BaseTest.getResource("new_contacts.csv")); + createdContactsAmount = contacts.getContacts().size(); + + contacts.getContacts().forEach(contact -> { + contact.setLastName(reference); + String mail = contact.getEmail(); + contact.setEmail(mail.replace(REPLACE_PATTERN, reference)); + }); + + client.createContacts(contacts); + } + + private static Stream> getContacts() throws IOException { + return client.getObject(ObjectHelper.getObjectInfo(MarketingContacts.class), null) + .stream() + .map(baseObject -> baseObject.asMap()) + .filter(fieldMap -> fieldMap.containsKey(LAST_NAME_FIELD) && fieldMap.get(LAST_NAME_FIELD).equals(reference)); + } + + @AfterClass + public static void tearDownTestClass() throws IOException { + List idsToRemove = getContacts() + .filter(fieldMap -> fieldMap.containsKey(ID_FIELD)) + .map(fieldMap -> (String) fieldMap.get(ID_FIELD)) + .filter(id -> id != null) + .collect(Collectors.toList()); + client.deleteContacts(idsToRemove); + } + + @Test + public void testBatchSource() throws Exception { + ImmutableMap.Builder optionsBuilder = new ImmutableMap.Builder<>(); + + if (authType == BASIC_AUTH_TYPE) { + optionsBuilder + .put(SendGridSourceConfig.PROPERTY_AUTH_TYPE, BASIC_AUTH_TYPE) + .put(SendGridSourceConfig.PROPERTY_AUTH_USERNAME, authUser) + .put(SendGridSourceConfig.PROPERTY_AUTH_PASSWORD, authPass); + } else { + optionsBuilder + .put(SendGridSourceConfig.PROPERTY_AUTH_TYPE, "api") + .put(SendGridSourceConfig.PROPERTY_SENDGRID_API_KEY, authToken); + + } + + optionsBuilder + .put("referenceName", "ref") + .put(SendGridSourceConfig.PROPERTY_DATA_SOURCE_TYPES, "MarketingCampaign") + .put(SendGridSourceConfig.PROPERTY_DATA_SOURCE_MARKETING, "Contacts") + .put(SendGridSourceConfig.PROPERTY_DATA_SOURCE_FIELDS, "created_at,email,first_name,last_name,updated_at"); + + ETLStage source = new ETLStage("name", new ETLPlugin(BaseConfig.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, + optionsBuilder.build(), null)); + ETLStage sink = new ETLStage("sink", MockSink.getPlugin("outputSink")); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(source) + .addStage(sink) + .addConnection(source.getName(), sink.getName()) + .build(); + + ApplicationId pipelineId = NamespaceId.DEFAULT.app("HttpBatch_"); + ApplicationManager appManager = deployApplication(pipelineId, new AppRequest<>(APP_ARTIFACT, etlConfig)); + + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.startAndWaitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); + + DataSetManager outputManager = getDataset("outputSink"); + List outputRecords = MockSink.readOutput(outputManager); + + int contactCount = client.getObject(ObjectHelper.getObjectInfo(MarketingContacts.class), null).size(); + int retrievedContactsCount = (int) outputRecords.stream() + .filter(x -> x.get("last_name").equals(reference)) + .count(); + + Assert.assertEquals(contactCount, outputRecords.size()); + Assert.assertEquals(getContacts().count(), retrievedContactsCount); + } +} diff --git a/src/test/resources/BaseConfigExample.json b/src/test/resources/BaseConfigExample.json new file mode 100644 index 0000000..9fca177 --- /dev/null +++ b/src/test/resources/BaseConfigExample.json @@ -0,0 +1,6 @@ +{ + "authType": "basic", + "sendGridApiKey": "", + "authUserName": "test user", + "authPassword": "test pass" +} \ No newline at end of file diff --git a/src/test/resources/KeyConfigExample.json b/src/test/resources/KeyConfigExample.json new file mode 100644 index 0000000..c4f2b72 --- /dev/null +++ b/src/test/resources/KeyConfigExample.json @@ -0,0 +1,6 @@ +{ + "authType": "api", + "sendGridApiKey": "some-api-key", + "authUserName": "", + "authPassword": "" +} \ No newline at end of file diff --git a/src/test/resources/SendGridSinkConfigExample.json b/src/test/resources/SendGridSinkConfigExample.json new file mode 100644 index 0000000..f4a93a2 --- /dev/null +++ b/src/test/resources/SendGridSinkConfigExample.json @@ -0,0 +1,15 @@ +{ + "from": "test@email.com", + "replyTo": "reply@email.com", + "footerEnable": "true", + "footerHTML": "footer", + "sandboxMode": "true", + "clickTracking": "true", + "openTracking": "true", + "subscriptionTracking": "true", + "recipientAddressSource": "input", + "recipientConfigAddressList": "test1@email.com,test2@email.com", + "recipientColumnName": "column1", + "bodyColumnName": "column2", + "mailSubject": "subject" +} \ No newline at end of file diff --git a/src/test/resources/SendGridSourceConfigExample.json b/src/test/resources/SendGridSourceConfigExample.json new file mode 100644 index 0000000..57f62d2 --- /dev/null +++ b/src/test/resources/SendGridSourceConfigExample.json @@ -0,0 +1,10 @@ +{ + "dataSourceTypes": "MarketingCampaign,Statistic,Suppression", + "dataSourceMarketing": "SingleSends,Senders", + "dataSourceStats": "CategoryStats", + "dataSourceSuppressions": "Bounces,GlobalUnsubscribes", + "dataSourceFields": "address,city,contracts_count", + "startDate": "2019-09-18", + "endDate": "2019-09-21", + "statCategories": "spam" +} \ No newline at end of file diff --git a/src/test/resources/new_contacts.csv b/src/test/resources/new_contacts.csv new file mode 100644 index 0000000..c2f1f82 --- /dev/null +++ b/src/test/resources/new_contacts.csv @@ -0,0 +1,4 @@ +user1.%id%@example.com,user1,user1,123 Neverland Lane,Suite 42,Denver,CO,80202,USA +user2.%id%@example.com,user2,user2,123 Neverland Lane,Suite 42,Denver,CO,80202,USA +user3.%id%@example.com,user3,user3,123 Neverland Lane,Suite 42,Denver,CO,80202,USA +user4.%id%@example.com,user4,user4,123 Neverland Lane,Suite 42,Denver,CO,80202,USA From 3d0d8eb672a2c54af0362eb2b317e4bb8cf5178e Mon Sep 17 00:00:00 2001 From: Dmitry Grinenko Date: Mon, 11 Nov 2019 13:46:32 +0200 Subject: [PATCH 6/6] minor fixes for the sink validation --- .../batch/sink/SendGridSinkConfig.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java index 126349b..777f6dc 100644 --- a/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java +++ b/src/main/java/io/cdap/plugin/sendgrid/batch/sink/SendGridSinkConfig.java @@ -163,35 +163,39 @@ public void validate(FailureCollector failureCollector) { new SendGridSinkConfigValidator(failureCollector, this).validate(); } - public void validate(Schema schema) { - if (schema == null) { - throw new IllegalArgumentException("Input schema cannot be empty"); - } - - if (getRecipientAddressSource() == ToAddressSource.INPUT) { - Schema.Field recipient = schema.getField(recipientColumnName); - if (recipient == null) { - throw new IllegalArgumentException(String.format("Plugin is configured to use column '%s' for" + + private void validateField(Schema.Field field, String name) { + if (field == null) { + throw new IllegalArgumentException(String.format("Plugin is configured to use column '%s' for" + " recipient addresses, but input schema did not provide such column", recipientColumnName)); - } + } - if (recipient.getSchema().getType() != Schema.Type.STRING) { + Schema fieldSchema = field.getSchema(); + if (fieldSchema.getType() == Schema.Type.UNION) { + if (fieldSchema.getUnionSchemas().stream().noneMatch(x -> x.getType() == Schema.Type.STRING)) { throw new IllegalArgumentException(String.format("The input schema column '%s' expected to be of type STRING", - recipientColumnName)); + name)); } + return; } - Schema.Field body = schema.getField(bodyColumnName); - if (body == null) { - throw new IllegalArgumentException(String.format("Plugin requires column '%s' for" + - " recipient addresses, but input schema did not provide such column", recipientColumnName)); - } - if (body.getSchema().getType() != Schema.Type.STRING) { + if (fieldSchema.getType() != Schema.Type.STRING) { throw new IllegalArgumentException(String.format("The input schema column '%s' expected to be of type STRING", - bodyColumnName)); + name)); } } + public void validate(Schema schema) { + if (schema == null) { + throw new IllegalArgumentException("Input schema cannot be empty"); + } + + if (getRecipientAddressSource() == ToAddressSource.INPUT) { + validateField(schema.getField(recipientColumnName), recipientColumnName); + } + + validateField(schema.getField(bodyColumnName), bodyColumnName); + } + public String getFrom() { if (Strings.isNullOrEmpty(from)) { throw new IllegalArgumentException(String.format("Property '%s' cannot be empty", PROPERTY_FROM));