#387 google ads connector implementation

httputils package that add a little convenience to handling http requests

.gitignore

@@ -14,3 +14,4 @@ ui-dev-compose/redis-insights
 .run/
 package-lock.json
 package.json
+server

configurator/backend/go.mod

@@ -18,8 +18,8 @@ require (
 	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/viper v1.8.1
 	golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
-	google.golang.org/api v0.44.0
-	google.golang.org/grpc v1.38.0
+	google.golang.org/api v0.56.0
+	google.golang.org/grpc v1.40.0
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/mail.v2 v2.3.1
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b

configurator/frontend/src/catalog/sources/lib/commonParams.tsx

@@ -12,7 +12,9 @@ export type GoogleParametersNodes = {
   type?: string
   disableOauth?: boolean
   disableServiceAccount?: boolean,
-  serviceAccountKey?: string
+  serviceAccountKey?: string,
+  requireSubject?: boolean,
+  subjectKey?: string
 }
 
 /**
@@ -34,7 +36,10 @@ export const googleAuthConfigParameters: (param?: GoogleParametersNodes) => Para
   type = 'config.auth.type',
   disableOauth = false,
   disableServiceAccount = false,
-  serviceAccountKey = 'config.auth.service_account_key'
+  requireSubject = false,
+  serviceAccountKey = 'config.auth.service_account_key',
+  subjectKey = 'config.auth.subject'
+
 }: GoogleParametersNodes = {}) => removeNulls([
   {
     displayName: 'Authorization Type',
@@ -104,5 +109,15 @@ export const googleAuthConfigParameters: (param?: GoogleParametersNodes) => Para
       <>
         <a href="https://cloud.google.com/iam/docs/creating-managing-service-account-keys">Use Google Cloud Console to create Service Account get Service Key JSON</a>
       </>
+  },
+  !disableServiceAccount && requireSubject && {
+    displayName: 'Subject',
+    id: subjectKey,
+    type: stringType,
+    required: true,
+    documentation:
+        <>
+        A Google Ads user with permissions on the Google Ads account you want to access. Google Ads does not support using service accounts without impersonation.
+        </>
   }
 ]);

configurator/frontend/src/catalog/sources/lib/native.tsx

@@ -1,4 +1,4 @@
-import { intType, passwordType, selectionType, SourceConnector, stringType } from '../types';
+import {intType, isoUtcDateType, passwordType, selectionType, SourceConnector, stringType} from '../types';
 import { googleServiceAuthDocumentation } from '../lib/documentation';
 
 import { googleAuthConfigParameters } from '../lib/commonParams';
@@ -146,6 +146,87 @@ export const facebook: SourceConnector = {
   ]
 };
 
+export const googleAds: SourceConnector = {
+  pic: (
+      <svg height="100%"
+           viewBox="0 0 192 192" width="100%" xmlns="http://www.w3.org/2000/svg">
+        <g className="_ngcontent-awn-AWSM-2">
+          <rect fill="none" height="192" width="192" className="_ngcontent-awn-AWSM-2"></rect>
+          <g className="_ngcontent-awn-AWSM-2">
+            <rect fill="#FBBC04" height="58.67" transform="matrix(0.5 -0.866 0.866 0.5 -46.2127 103.666)" width="117.33"
+                  x="8" y="62.52" className="_ngcontent-awn-AWSM-2"></rect>
+            <path
+                d="M180.07,127.99L121.4,26.38c-8.1-14.03-26.04-18.84-40.07-10.74c-14.03,8.1-18.84,26.04-10.74,40.07 l58.67,101.61c8.1,14.03,26.04,18.83,40.07,10.74C183.36,159.96,188.16,142.02,180.07,127.99z"
+                fill="#4285F4" className="_ngcontent-awn-AWSM-2"></path>
+            <circle cx="37.34" cy="142.66" fill="#34A853" r="29.33" className="_ngcontent-awn-AWSM-2"></circle>
+          </g>
+        </g>
+      </svg>
+  ),
+  collectionParameters: [
+    {
+      displayName: 'Fields',
+      documentation: (
+          <> Use <a href="https://developers.google.com/google-ads/api/fields/v8/overview_query_builder">Google Ads Query Builder</a> tool to build required query. Copy comma-separated field list from resulting GAQL query (part between SELECT and FROM keywords).
+            Don't forget to add date segments (e.g. segments.date) where applicable.
+          </>
+      ),
+      id: 'fields',
+      // prettier-ignore
+      type: stringType
+    },
+    {
+      displayName: 'Start Date',
+      id: 'start_date',
+      type: isoUtcDateType,
+      defaultValue: '2018-12-31',
+      required: true
+    }
+  ],
+  collectionTemplates: [
+  ],
+
+  displayName: 'Google Ads',
+  id: 'google_ads',
+  collectionTypes: ['accessible_bidding_strategy','account_budget','account_budget_proposal','account_link','ad_group','ad_group_ad','ad_group_ad_asset_view','ad_group_ad_label','ad_group_asset','ad_group_audience_view','ad_group_bid_modifier','ad_group_criterion','ad_group_criterion_label','ad_group_criterion_simulation','ad_group_extension_setting','ad_group_feed','ad_group_label','ad_group_simulation','ad_parameter','ad_schedule_view','age_range_view','asset','asset_field_type_view','batch_job','bidding_data_exclusion','bidding_seasonality_adjustment','bidding_strategy','bidding_strategy_simulation','billing_setup','call_view','campaign','campaign_asset','campaign_audience_view','campaign_bid_modifier','campaign_budget','campaign_criterion','campaign_criterion_simulation','campaign_draft','campaign_experiment','campaign_extension_setting','campaign_feed','campaign_label','campaign_shared_set','campaign_simulation','carrier_constant','change_event','change_status','click_view','combined_audience','conversion_action','conversion_custom_variable','conversion_value_rule','conversion_value_rule_set','currency_constant','custom_audience','custom_interest','customer','customer_asset','customer_client','customer_client_link','customer_extension_setting','customer_feed','customer_label','customer_manager_link','customer_negative_criterion','customer_user_access','customer_user_access_invitation','detail_placement_view','detailed_demographic','display_keyword_view','distance_view','domain_category','dynamic_search_ads_search_term_view','expanded_landing_page_view','extension_feed_item','feed','feed_item','feed_item_set','feed_item_set_link','feed_item_target','feed_mapping','feed_placeholder_view','gender_view','geo_target_constant','geographic_view','group_placement_view','hotel_group_view','hotel_performance_view','income_range_view','keyword_plan','keyword_plan_ad_group','keyword_plan_ad_group_keyword','keyword_plan_campaign','keyword_plan_campaign_keyword','keyword_theme_constant','keyword_view','label','landing_page_view','language_constant','life_event','location_view','managed_placement_view','media_file','mobile_app_category_constant','mobile_device_constant','offline_user_data_job','operating_system_version_constant','paid_organic_search_term_view','parental_status_view','product_bidding_category_constant','product_group_view','recommendation','remarketing_action','search_term_view','shared_criterion','shared_set','shopping_performance_view','smart_campaign_search_term_view','smart_campaign_setting','third_party_app_analytics_link','topic_constant','topic_view','user_interest','user_list','user_location_view','video','webpage_view'],
+  configParameters: [
+    ...googleAuthConfigParameters({
+      requireSubject: true
+    }),
+    {
+      displayName: 'Customer ID',
+      id: 'config.customer_id',
+      type: stringType,
+      required: true
+    },{
+      displayName: 'Manager Customer ID',
+      id: 'config.manager_customer_id',
+      type: stringType,
+      required: false,
+      documentation: (
+          <>
+            For Google Ads API calls made by a manager to a client account (that is, when logging in as a manager to make API calls to one of its client accounts), you also need to supply the Manager Customer Id. This value represents the Google Ads customer ID of the manager making the API call.
+          </>
+      )
+    }
+  ],
+  documentation: {
+    overview: (
+        <>
+          The Google Ads
+        </>
+    ),
+    connection: googleServiceAuthDocumentation({
+      oauthEnabled: true,
+      serviceAccountEnabled: true,
+      scopes: ['https://www.googleapis.com/auth/adwords.readonly'],
+      serviceName: 'Google Ads',
+      apis: ['Google Ads API']
+    })
+  }
+};
+
+
 export const googleAnalytics: SourceConnector = {
   pic: (
     <svg
@@ -706,4 +787,4 @@ export const amplitude: SourceConnector = {
   collectionParameters: []
 };
 
-export const allNativeConnectors = [facebook, redis, firebase, googleAnalytics, googlePlay, amplitude];
+export const allNativeConnectors = [facebook, redis, firebase, googleAds, googleAnalytics, googlePlay, amplitude];

configurator/frontend/src/catalog/sources/types.ts

@@ -168,6 +168,7 @@ export interface CollectionParameter extends Parameter {
 
 type SourceConnectorId = 
   | "facebook_marketing"
+  | "google_ads"
   | "google_analytics"
   | "google_play"
   | "firebase"
@@ -178,6 +179,10 @@ export interface SourceConnector {
    * Is it singer source or not, optional parameter.
    * */
   isSingerType?: boolean;
+  /**
+   * Enable collection Start Date parameter.
+   * */
+  isStartDateEnabled?: boolean;
 
   /**
    * If connector requires expert-level knowledge (such as JSON editing)

server/drivers/base/config.go

@@ -2,7 +2,10 @@ package base
 
 import (
 	"errors"
+	"fmt"
 	"github.com/jitsucom/jitsu/server/logging"
+	"github.com/jitsucom/jitsu/server/timestamp"
+	"time"
 )
 
 //SourceConfig is a dto for api connector source config serialization
@@ -32,6 +35,20 @@ type Collection struct {
 	Parameters   map[string]interface{} `mapstructure:"parameters" json:"parameters,omitempty" yaml:"parameters,omitempty"`
 }
 
+func (c *Collection) Init() error  {
+	if c.StartDateStr != "" {
+		startDate, err := time.Parse(timestamp.DashDayLayout, c.StartDateStr)
+		if err != nil {
+			return fmt.Errorf("Malformed start_date in [%s_%s] collection: please use YYYY-MM-DD format: %v", c.SourceID, c.Name, err)
+		}
+
+		date := time.Date(startDate.Year(), startDate.Month(), startDate.Day(), 0, 0, 0, 0, time.UTC)
+		c.DaysBackToLoad = getDaysBackToLoad(&date)
+		logging.Infof("[%s_%s] Using start date: %s", c.SourceID, c.Name, date)
+	}
+	return nil
+}
+
 //Validate returns err if collection invalid
 func (c *Collection) Validate() error {
 	if c.Name == "" {
@@ -53,3 +70,11 @@ func (c *Collection) GetTableName() string {
 	}
 	return c.SourceID + "_" + c.Name
 }
+
+//return difference between now and t in DAYS + 1 (current day)
+//e.g. 2021-03-01 - 2021-03-01 = 0, but we should load current date as well
+func getDaysBackToLoad(t *time.Time) int {
+	now := time.Now().UTC()
+	currentDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
+	return int(currentDay.Sub(*t).Hours()/24) + 1
+}

server/drivers/base/granularity.go

@@ -10,7 +10,9 @@ type Granularity string
 
 const (
 	DAY   Granularity = "DAY"
+	WEEK  Granularity = "WEEK"
 	MONTH Granularity = "MONTH"
+	QUARTER Granularity = "QUARTER"
 	YEAR  Granularity = "YEAR"
 	ALL   Granularity = "ALL"
 )
@@ -19,9 +21,13 @@ const (
 func (g Granularity) Lower(t time.Time) time.Time {
 	switch g {
 	case DAY:
-		return t.Truncate(time.Hour * 24)
+		return t.UTC().Truncate(time.Hour * 24)
+	case WEEK:
+		return t.UTC().Truncate(time.Hour * 24 * 7)
 	case MONTH:
 		return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, t.Location())
+	case QUARTER:
+		return time.Date(t.Year(), t.Month() - (t.Month()-1)%3, 1, 0, 0, 0, 0, t.Location())
 	case YEAR:
 		return time.Date(t.Year(), 1, 1, 0, 0, 0, 0, t.Location())
 	case ALL:
@@ -37,8 +43,12 @@ func (g Granularity) Upper(t time.Time) time.Time {
 	switch g {
 	case DAY:
 		return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()).AddDate(0, 0, 1).Add(-time.Nanosecond)
+	case WEEK:
+		return t.UTC().Truncate(time.Hour * 24 * 7).AddDate(0, 0, 7).Add(-time.Nanosecond)
 	case MONTH:
 		return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, t.Location()).AddDate(0, 1, 0).Add(-time.Nanosecond)
+	case QUARTER:
+		return time.Date(t.Year(), t.Month() - (t.Month()-1)%3, 1, 0, 0, 0, 0, t.Location()).AddDate(0, 3, 0).Add(-time.Nanosecond)
 	case YEAR:
 		return time.Date(t.Year(), 1, 1, 0, 0, 0, 0, t.Location()).AddDate(1, 0, 0).Add(-time.Nanosecond)
 	case ALL:
@@ -54,8 +64,12 @@ func (g Granularity) Format(t time.Time) string {
 	switch g {
 	case DAY:
 		return t.Format("2006-01-02")
+	case WEEK:
+		return t.Format("2006-01-02")
 	case MONTH:
 		return t.Format("2006-01")
+	case QUARTER:
+		return t.Format("2006-01")
 	case YEAR:
 		return t.Format("2006")
 	case ALL:
@@ -71,8 +85,12 @@ func (g Granularity) String() string {
 	switch g {
 	case DAY:
 		return string(DAY)
+	case WEEK:
+		return string(WEEK)
 	case MONTH:
 		return string(MONTH)
+	case QUARTER:
+		return string(QUARTER)
 	case YEAR:
 		return string(YEAR)
 	case ALL:

server/drivers/base/types.go

@@ -16,6 +16,7 @@ const (
 	FirebaseType        = "firebase"
 	GoogleAnalyticsType = "google_analytics"
 	GooglePlayType      = "google_play"
+	GoogleAdsType 		= "google_ads"
 	RedisType           = "redis"
 
 	SingerType = "singer"
@@ -37,6 +38,7 @@ type GoogleAuthConfig struct {
 	ClientSecret      string      `mapstructure:"client_secret" json:"client_secret,omitempty" yaml:"client_secret,omitempty"`
 	RefreshToken      string      `mapstructure:"refresh_token" json:"refresh_token,omitempty" yaml:"refresh_token,omitempty"`
 	ServiceAccountKey interface{} `mapstructure:"service_account_key" json:"service_account_key,omitempty" yaml:"service_account_key,omitempty"`
+	Subject			  string      `mapstructure:"subject" json:"subject,omitempty" yaml:"subject,omitempty"`
 }
 
 func (gac *GoogleAuthConfig) Marshal() ([]byte, error) {

server/drivers/factory.go

@@ -4,19 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
-
 	_ "github.com/jitsucom/jitsu/server/drivers/amplitude"
 	"github.com/jitsucom/jitsu/server/drivers/base"
 	_ "github.com/jitsucom/jitsu/server/drivers/facebook_marketing"
 	_ "github.com/jitsucom/jitsu/server/drivers/firebase"
+	_ "github.com/jitsucom/jitsu/server/drivers/google_ads"
 	_ "github.com/jitsucom/jitsu/server/drivers/google_analytics"
 	_ "github.com/jitsucom/jitsu/server/drivers/google_play"
 	_ "github.com/jitsucom/jitsu/server/drivers/redis"
 	_ "github.com/jitsucom/jitsu/server/drivers/singer"
 	"github.com/jitsucom/jitsu/server/logging"
 	"github.com/jitsucom/jitsu/server/scheduling"
-	"github.com/jitsucom/jitsu/server/timestamp"
 	"github.com/spf13/cast"
 )
 
@@ -50,19 +48,6 @@ func Create(ctx context.Context, sourceID string, sourceConfig *base.SourceConfi
 		return nil, errors.New("collections are empty. Please specify at least one collection")
 	}
 
-	for _, collection := range collections {
-		if collection.StartDateStr != "" {
-			startDate, err := time.Parse(timestamp.DashDayLayout, collection.StartDateStr)
-			if err != nil {
-				return nil, fmt.Errorf("Malformed start_date in %s collection: please use YYYY-MM-DD format: %v", collection.Name, err)
-			}
-
-			date := time.Date(startDate.Year(), startDate.Month(), startDate.Day(), 0, 0, 0, 0, time.UTC)
-			collection.DaysBackToLoad = getDaysBackToLoad(&date)
-			logging.Infof("[%s_%s] Using start date: %s", sourceID, collection.Name, date)
-		}
-	}
-
 	driverPerCollection := map[string]base.Driver{}
 
 	createDriverFunc, ok := base.DriverConstructors[sourceConfig.Type]
@@ -145,7 +130,9 @@ func ParseCollections(sourceConfig *base.SourceConfig) ([]*base.Collection, erro
 			}
 
 			collectionObj.SourceID = sourceConfig.SourceID
-
+			if err := collectionObj.Init(); err != nil {
+				return nil, err
+			}
 			if err := collectionObj.Validate(); err != nil {
 				return nil, err
 			}
@@ -158,11 +145,3 @@ func ParseCollections(sourceConfig *base.SourceConfig) ([]*base.Collection, erro
 
 	return collections, nil
 }
-
-//return difference between now and t in DAYS + 1 (current day)
-//e.g. 2021-03-01 - 2021-03-01 = 0, but we should load current date as well
-func getDaysBackToLoad(t *time.Time) int {
-	now := time.Now().UTC()
-	currentDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
-	return int(currentDay.Sub(*t).Hours()/24) + 1
-}