feat(hubspot): Extract associations (#10919)

#### Summary  Extracts the HubSpot objects associations and adds their JSON content as a column in the extracted table Relates to #10915 <!--
cloudquery · May 25, 2023 · a4e1ff1 · a4e1ff1
1 parent eaad217
commit a4e1ff1
Show file tree

Hide file tree

Showing 24 changed files with 48 additions and 15 deletions.
diff --git a/plugins/source/hubspot/client/spec.go b/plugins/source/hubspot/client/spec.go
@@ -8,7 +8,8 @@ type Spec struct {
 type TableOptions map[string]*TableOptionsSpec
 
 type TableOptionsSpec struct {
-	Properties []string `yaml:"properties,omitempty" json:"properties,omitempty"`
+	Properties   []string `yaml:"properties,omitempty" json:"properties,omitempty"`
+	Associations []string `yaml:"associations,omitempty" json:"associations,omitempty"`
 }
 
 func (spec *Spec) setDefaults() {
@@ -35,3 +36,10 @@ func (to *TableOptionsSpec) GetProperties() []string {
 	}
 	return to.Properties
 }
+
+func (to *TableOptionsSpec) GetAssociations() []string {
+	if to == nil {
+		return nil
+	}
+	return to.Associations
+}
diff --git a/plugins/source/hubspot/resources/services/crm/companies.go b/plugins/source/hubspot/resources/services/crm/companies.go
@@ -14,7 +14,7 @@ func Companies() *schema.Table {
 		Transform: transformers.TransformWithStruct(
 			companies.SimplePublicObjectWithAssociations{},
 			transformers.WithPrimaryKeys("Id"),
-			transformers.WithSkipFields("PropertiesWithHistory", "Associations"),
+			transformers.WithSkipFields("PropertiesWithHistory"),
 		),
 	}
 }
diff --git a/plugins/source/hubspot/resources/services/crm/companies_fetch.go b/plugins/source/hubspot/resources/services/crm/companies_fetch.go
@@ -22,6 +22,7 @@ func fetchCompanies(ctx context.Context, meta schema.ClientMeta, parent *schema.
 		req := hubspotClient.BasicApi.
 			GetPage(hubspot.WithAuthorizer(ctx, cqClient.Authorizer)).
 			Properties(cqClient.Spec.TableOptions.ForTable("hubspot_crm_companies").GetProperties()).
+			Associations(cqClient.Spec.TableOptions.ForTable("hubspot_crm_companies").GetAssociations()).
 			Limit(client.DefaultPageSize)
 
 		if len(after) > 0 {

diff --git a/plugins/source/hubspot/resources/services/crm/contacts.go b/plugins/source/hubspot/resources/services/crm/contacts.go
@@ -13,7 +13,7 @@ func Contacts() *schema.Table {
 		Transform: transformers.TransformWithStruct(
 			contacts.SimplePublicObjectWithAssociations{},
 			transformers.WithPrimaryKeys("Id"),
-			transformers.WithSkipFields("PropertiesWithHistory", "Associations"),
+			transformers.WithSkipFields("PropertiesWithHistory"),
 		),
 		Resolver: fetchContacts,
 	}

diff --git a/plugins/source/hubspot/resources/services/crm/contacts_fetch.go b/plugins/source/hubspot/resources/services/crm/contacts_fetch.go
@@ -24,6 +24,7 @@ func fetchContacts(ctx context.Context, meta schema.ClientMeta, parent *schema.R
 		req := hubspotClient.BasicApi.
 			GetPage(hubspot.WithAuthorizer(ctx, cqClient.Authorizer)).
 			Properties(cqClient.Spec.TableOptions.ForTable("hubspot_crm_contacts").GetProperties()).
+			Associations(cqClient.Spec.TableOptions.ForTable("hubspot_crm_contacts").GetAssociations()).
 			Limit(client.DefaultPageSize)
 
 		if len(after) > 0 {

diff --git a/plugins/source/hubspot/resources/services/crm/deals.go b/plugins/source/hubspot/resources/services/crm/deals.go
@@ -14,7 +14,7 @@ func Deals() *schema.Table {
 		Transform: transformers.TransformWithStruct(
 			deals.SimplePublicObjectWithAssociations{},
 			transformers.WithPrimaryKeys("Id"),
-			transformers.WithSkipFields("PropertiesWithHistory", "Associations"),
+			transformers.WithSkipFields("PropertiesWithHistory"),
 		),
 	}
 }
diff --git a/plugins/source/hubspot/resources/services/crm/deals_fetch.go b/plugins/source/hubspot/resources/services/crm/deals_fetch.go
@@ -22,6 +22,7 @@ func fetchDeals(ctx context.Context, meta schema.ClientMeta, parent *schema.Reso
 		req := hubspotClient.BasicApi.
 			GetPage(hubspot.WithAuthorizer(ctx, cqClient.Authorizer)).
 			Properties(cqClient.Spec.TableOptions.ForTable("hubspot_crm_deals").GetProperties()).
+			Associations(cqClient.Spec.TableOptions.ForTable("hubspot_crm_deals").GetAssociations()).
 			Limit(client.DefaultPageSize)
 
 		if len(after) > 0 {

diff --git a/plugins/source/hubspot/resources/services/crm/line_items.go b/plugins/source/hubspot/resources/services/crm/line_items.go
@@ -14,7 +14,7 @@ func LineItems() *schema.Table {
 		Transform: transformers.TransformWithStruct(
 			line_items.SimplePublicObjectWithAssociations{},
 			transformers.WithPrimaryKeys("Id"),
-			transformers.WithSkipFields("PropertiesWithHistory", "Associations"),
+			transformers.WithSkipFields("PropertiesWithHistory"),
 		),
 	}
 }
diff --git a/plugins/source/hubspot/resources/services/crm/line_items_fetch.go b/plugins/source/hubspot/resources/services/crm/line_items_fetch.go
@@ -22,6 +22,7 @@ func fetchLineItems(ctx context.Context, meta schema.ClientMeta, parent *schema.
 		req := hubspotClient.BasicApi.
 			GetPage(hubspot.WithAuthorizer(ctx, cqClient.Authorizer)).
 			Properties(cqClient.Spec.TableOptions.ForTable("hubspot_crm_line_items").GetProperties()).
+			Associations(cqClient.Spec.TableOptions.ForTable("hubspot_crm_line_items").GetAssociations()).
 			Limit(client.DefaultPageSize)
 
 		if len(after) > 0 {

diff --git a/plugins/source/hubspot/resources/services/crm/products.go b/plugins/source/hubspot/resources/services/crm/products.go
@@ -14,7 +14,7 @@ func Products() *schema.Table {
 		Transform: transformers.TransformWithStruct(
 			products.SimplePublicObjectWithAssociations{},
 			transformers.WithPrimaryKeys("Id"),
-			transformers.WithSkipFields("PropertiesWithHistory", "Associations"),
+			transformers.WithSkipFields("PropertiesWithHistory"),
 		),
 	}
 }
diff --git a/plugins/source/hubspot/resources/services/crm/products_fetch.go b/plugins/source/hubspot/resources/services/crm/products_fetch.go
@@ -22,6 +22,7 @@ func fetchProducts(ctx context.Context, meta schema.ClientMeta, parent *schema.R
 		req := hubspotClient.BasicApi.
 			GetPage(hubspot.WithAuthorizer(ctx, cqClient.Authorizer)).
 			Properties(cqClient.Spec.TableOptions.ForTable("hubspot_crm_products").GetProperties()).
+			Associations(cqClient.Spec.TableOptions.ForTable("hubspot_crm_products").GetAssociations()).
 			Limit(client.DefaultPageSize)
 
 		if len(after) > 0 {

diff --git a/plugins/source/hubspot/resources/services/crm/quotes.go b/plugins/source/hubspot/resources/services/crm/quotes.go
@@ -14,7 +14,7 @@ func Quotes() *schema.Table {
 		Transform: transformers.TransformWithStruct(
 			quotes.SimplePublicObjectWithAssociations{},
 			transformers.WithPrimaryKeys("Id"),
-			transformers.WithSkipFields("PropertiesWithHistory", "Associations"),
+			transformers.WithSkipFields("PropertiesWithHistory"),
 		),
 	}
 }
diff --git a/plugins/source/hubspot/resources/services/crm/quotes_fetch.go b/plugins/source/hubspot/resources/services/crm/quotes_fetch.go
@@ -22,6 +22,7 @@ func fetchQuotes(ctx context.Context, meta schema.ClientMeta, parent *schema.Res
 		req := hubspotClient.BasicApi.
 			GetPage(hubspot.WithAuthorizer(ctx, cqClient.Authorizer)).
 			Properties(cqClient.Spec.TableOptions.ForTable("hubspot_crm_quotes").GetProperties()).
+			Associations(cqClient.Spec.TableOptions.ForTable("hubspot_crm_quotes").GetAssociations()).
 			Limit(client.DefaultPageSize)
 
 		if len(after) > 0 {

diff --git a/plugins/source/hubspot/resources/services/crm/tickets.go b/plugins/source/hubspot/resources/services/crm/tickets.go
@@ -14,7 +14,7 @@ func Tickets() *schema.Table {
 		Transform: transformers.TransformWithStruct(
 			tickets.SimplePublicObjectWithAssociations{},
 			transformers.WithPrimaryKeys("Id"),
-			transformers.WithSkipFields("PropertiesWithHistory", "Associations"),
+			transformers.WithSkipFields("PropertiesWithHistory"),
 		),
 	}
 }
diff --git a/plugins/source/hubspot/resources/services/crm/tickets_fetch.go b/plugins/source/hubspot/resources/services/crm/tickets_fetch.go
@@ -22,6 +22,7 @@ func fetchTickets(ctx context.Context, meta schema.ClientMeta, parent *schema.Re
 		req := hubspotClient.BasicApi.
 			GetPage(hubspot.WithAuthorizer(ctx, cqClient.Authorizer)).
 			Properties(cqClient.Spec.TableOptions.ForTable("hubspot_crm_tickets").GetProperties()).
+			Associations(cqClient.Spec.TableOptions.ForTable("hubspot_crm_tickets").GetAssociations()).
 			Limit(client.DefaultPageSize)
 
 		if len(after) > 0 {

diff --git a/website/pages/docs/plugins/sources/hubspot/_configuration.mdx b/website/pages/docs/plugins/sources/hubspot/_configuration.mdx
@@ -12,17 +12,24 @@ spec:
     max_requests_per_second: 5
     table_options:
       hubspot_crm_companies:
+        associations: []
         properties: []
       hubspot_crm_contacts:
+        associations: []
         properties: []
       hubspot_crm_deals:
+        associations: []
         properties: []
       hubspot_crm_line_items:
+        associations: []
         properties: []
       hubspot_crm_products:
+        associations: []
         properties: []
       hubspot_crm_quotes:
+        associations: []
         properties: []
       hubspot_crm_tickets:
+        associations: []
         properties: []
 ```
diff --git a/website/pages/docs/plugins/sources/hubspot/overview.mdx b/website/pages/docs/plugins/sources/hubspot/overview.mdx
@@ -45,6 +45,10 @@ This is the specs that can be used by the HubSpot source Plugin.
 
 ### Table Options
 
+- `associations` (`[]string`, optional. Default: empty)
+
+  Additional associations to be retrieved from HubSpot when syncing the table entity
+
 - `properties` (`[]string`, optional. Default: empty)
 
   Additional properties to be retrieved from HubSpot when syncing the table entity
diff --git a/website/tables/hubspot/hubspot_crm_companies.md b/website/tables/hubspot/hubspot_crm_companies.md
@@ -19,4 +19,5 @@ The primary key for this table is **id**.
 |created_at|Timestamp|
 |updated_at|Timestamp|
 |archived|Bool|
-|archived_at|Timestamp|
+|archived_at|Timestamp|
+|associations|JSON|
diff --git a/website/tables/hubspot/hubspot_crm_contacts.md b/website/tables/hubspot/hubspot_crm_contacts.md
@@ -19,4 +19,5 @@ The primary key for this table is **id**.
 |created_at|Timestamp|
 |updated_at|Timestamp|
 |archived|Bool|
-|archived_at|Timestamp|
+|archived_at|Timestamp|
+|associations|JSON|
diff --git a/website/tables/hubspot/hubspot_crm_deals.md b/website/tables/hubspot/hubspot_crm_deals.md
@@ -19,4 +19,5 @@ The primary key for this table is **id**.
 |created_at|Timestamp|
 |updated_at|Timestamp|
 |archived|Bool|
-|archived_at|Timestamp|
+|archived_at|Timestamp|
+|associations|JSON|
diff --git a/website/tables/hubspot/hubspot_crm_line_items.md b/website/tables/hubspot/hubspot_crm_line_items.md
@@ -19,4 +19,5 @@ The primary key for this table is **id**.
 |created_at|Timestamp|
 |updated_at|Timestamp|
 |archived|Bool|
-|archived_at|Timestamp|
+|archived_at|Timestamp|
+|associations|JSON|
diff --git a/website/tables/hubspot/hubspot_crm_products.md b/website/tables/hubspot/hubspot_crm_products.md
@@ -19,4 +19,5 @@ The primary key for this table is **id**.
 |created_at|Timestamp|
 |updated_at|Timestamp|
 |archived|Bool|
-|archived_at|Timestamp|
+|archived_at|Timestamp|
+|associations|JSON|
diff --git a/website/tables/hubspot/hubspot_crm_quotes.md b/website/tables/hubspot/hubspot_crm_quotes.md
@@ -19,4 +19,5 @@ The primary key for this table is **id**.
 |created_at|Timestamp|
 |updated_at|Timestamp|
 |archived|Bool|
-|archived_at|Timestamp|
+|archived_at|Timestamp|
+|associations|JSON|
diff --git a/website/tables/hubspot/hubspot_crm_tickets.md b/website/tables/hubspot/hubspot_crm_tickets.md
@@ -19,4 +19,5 @@ The primary key for this table is **id**.
 |created_at|Timestamp|
 |updated_at|Timestamp|
 |archived|Bool|
-|archived_at|Timestamp|
+|archived_at|Timestamp|
+|associations|JSON|