From efddff5785367b9c51f24ad2f5af399f9f4f8f02 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Wed, 16 Jul 2025 00:04:49 -0700 Subject: [PATCH 1/8] feat: Init salesforce sync --- Makefile | 14 +- cmd/ctrlc/root/sync/salesforce/README.md | 264 +++++++++++ .../root/sync/salesforce/accounts/accounts.go | 286 ++++++++++++ .../root/sync/salesforce/common/client.go | 41 ++ cmd/ctrlc/root/sync/salesforce/common/util.go | 430 ++++++++++++++++++ .../salesforce/opportunities/opportunities.go | 273 +++++++++++ cmd/ctrlc/root/sync/salesforce/salesforce.go | 25 + cmd/ctrlc/root/sync/sync.go | 2 + go.mod | 18 +- go.sum | 36 +- 10 files changed, 1367 insertions(+), 22 deletions(-) create mode 100644 cmd/ctrlc/root/sync/salesforce/README.md create mode 100644 cmd/ctrlc/root/sync/salesforce/accounts/accounts.go create mode 100644 cmd/ctrlc/root/sync/salesforce/common/client.go create mode 100644 cmd/ctrlc/root/sync/salesforce/common/util.go create mode 100644 cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go create mode 100644 cmd/ctrlc/root/sync/salesforce/salesforce.go diff --git a/Makefile b/Makefile index 0e506cb..9bc2f40 100644 --- a/Makefile +++ b/Makefile @@ -5,18 +5,22 @@ LDFLAGS = -X github.com/ctrlplanedev/cli/cmd/ctrlc/root/version.Version=$(VERSIO -X github.com/ctrlplanedev/cli/cmd/ctrlc/root/version.GitCommit=$(COMMIT) \ -X github.com/ctrlplanedev/cli/cmd/ctrlc/root/version.BuildDate=$(DATE) -.PHONY: build build: go build -ldflags "$(LDFLAGS)" -o bin/ctrlc ./cmd/ctrlc -.PHONY: install install: go install -ldflags "$(LDFLAGS)" ./cmd/ctrlc -.PHONY: test test: go test -v ./... -.PHONY: clean clean: - rm -rf bin/ \ No newline at end of file + rm -rf bin/ + +lint: + golangci-lint run ./... + +format: + go fmt ./... + +.PHONY: build install test clean lint format \ No newline at end of file diff --git a/cmd/ctrlc/root/sync/salesforce/README.md b/cmd/ctrlc/root/sync/salesforce/README.md new file mode 100644 index 0000000..733f63a --- /dev/null +++ b/cmd/ctrlc/root/sync/salesforce/README.md @@ -0,0 +1,264 @@ +# Salesforce Sync + +This package provides functionality to sync Salesforce CRM data into Ctrlplane as resources. + +## Usage + +### Prerequisites + +You need Salesforce OAuth2 credentials: +- **Domain**: Your Salesforce instance URL (e.g., `https://my-domain.my.salesforce.com`) +- **Consumer Key**: From your Salesforce Connected App +- **Consumer Secret**: From your Salesforce Connected App + +### Setting up Salesforce Connected App + +1. Go to Setup → Apps → App Manager +2. Click "New Connected App" +3. Fill in the required fields +4. Enable OAuth Settings +5. Add OAuth Scopes: + - `api` - Access and manage your data + - `refresh_token` - Perform requests on your behalf at any time +6. Save and note the Consumer Key and Consumer Secret + +### Authentication + +You can provide credentials via environment variables: + +```bash +export SALESFORCE_DOMAIN="https://my-domain.my.salesforce.com" +export SALESFORCE_CONSUMER_KEY="your-consumer-key" +export SALESFORCE_CONSUMER_SECRET="your-consumer-secret" +``` + +Or via command-line flags: + +```bash +ctrlc sync salesforce accounts \ + --domain "https://my-domain.my.salesforce.com" \ + --consumer-key "your-consumer-key" \ + --consumer-secret "your-consumer-secret" +``` + +### Command-Line Flags + +Both `accounts` and `opportunities` commands support the following flags: + +| Flag | Description | Default | +|------|-------------|---------| +| `--domain` | Salesforce instance URL | `$SALESFORCE_DOMAIN` | +| `--consumer-key` | OAuth2 consumer key | `$SALESFORCE_CONSUMER_KEY` | +| `--consumer-secret` | OAuth2 consumer secret | `$SALESFORCE_CONSUMER_SECRET` | +| `--provider`, `-p` | Resource provider name | `salesforce-accounts` or `salesforce-opportunities` | +| `--metadata` | Custom metadata mappings (can be used multiple times) | Built-in defaults | +| `--where` | SOQL WHERE clause to filter records | None (syncs all records) | +| `--limit` | Maximum number of records to sync | 0 (no limit) | +| `--list-all-fields` | Log all available Salesforce fields | false | + +### Syncing Accounts + +```bash +# Sync all Salesforce accounts +ctrlc sync salesforce accounts + +# Sync accounts with a filter (e.g., only accounts with Customer Health populated) +ctrlc sync salesforce accounts --where="Customer_Health__c != null" + +# Sync accounts with complex filters +ctrlc sync salesforce accounts --where="Type = 'Customer' AND AnnualRevenue > 1000000" + +# Sync accounts and list all available fields in logs +ctrlc sync salesforce accounts --list-all-fields + +# Sync with custom provider name +ctrlc sync salesforce accounts --provider my-salesforce-accounts + +# Limit the number of records to sync +ctrlc sync salesforce accounts --limit 500 + +# Combine filters with metadata mappings +ctrlc sync salesforce accounts \ + --where="Industry = 'Technology'" \ + --metadata="account/revenue=AnnualRevenue" +``` + +#### Custom Field Mappings + +You can map any Salesforce field (including custom fields) to Ctrlplane metadata: + +```bash +# Map standard and custom fields to metadata +ctrlc sync salesforce accounts \ + --metadata="account/id=Id" \ + --metadata="ctrlplane/external-id=MasterRecordId" \ + --metadata="account/owner-id=OwnerId" \ + --metadata="account/tier=Tier__c" \ + --metadata="account/region=Region__c" \ + --metadata="account/annual-revenue=Annual_Revenue__c" \ + --metadata="account/health=Customer_Health__c" +``` + +**Note**: +- Metadata values are always stored as strings, so all field values are automatically converted +- The format is `ctrlplane/key=SalesforceField` for metadata mappings +- The sync automatically includes common fields with default mappings that can be overridden +- The `MasterRecordId` is mapped to `ctrlplane/external-id` by default +- All fields are fetched from Salesforce, so custom fields are always available for mapping + +### Syncing Opportunities + +```bash +# Sync all Salesforce opportunities +ctrlc sync salesforce opportunities + +# Sync only open opportunities +ctrlc sync salesforce opportunities --where="IsClosed = false" + +# Sync opportunities with complex filters +ctrlc sync salesforce opportunities --where="Amount > 50000 AND StageName != 'Closed Lost'" + +# Sync opportunities and list all available fields in logs +ctrlc sync salesforce opportunities --list-all-fields + +# Sync with custom provider name +ctrlc sync salesforce opportunities --provider my-salesforce-opportunities + +# Limit the number of records to sync +ctrlc sync salesforce opportunities --limit 500 + +# Combine filters with metadata mappings +ctrlc sync salesforce opportunities \ + --where="Amount > 100000" \ + --metadata="opportunity/probability=Probability" +``` + +#### Custom Field Mappings + +Just like accounts, you can map any Salesforce opportunity field (including custom fields) to Ctrlplane metadata: + +```bash +# Map standard and custom fields to metadata +ctrlc sync salesforce opportunities \ + --metadata="opportunity/id=Id" \ + --metadata="opportunity/account-id=AccountId" \ + --metadata="opportunity/type=Type__c" \ + --metadata="opportunity/expected-revenue=ExpectedRevenue" \ + --metadata="opportunity/lead-source=LeadSource" +``` + +**Note**: +- Metadata values are always stored as strings, so all field values are automatically converted +- The format is `ctrlplane/key=SalesforceField` for metadata mappings +- The sync automatically includes common fields with default mappings that can be overridden + +## Resource Schema + +### Salesforce Account Resource + +Resources are created with the following structure: + +```json +{ + "version": "ctrlplane.dev/crm/account/v1", + "kind": "SalesforceAccount", + "name": "Account Name", + "identifier": "001XX000003DHPh", + "config": { + "name": "Account Name", + "industry": "Technology", + "id": "001XX000003DHPh", + "salesforceAccount": { + "recordId": "001XX000003DHPh", + "ownerId": "005XX000001SvogAAC", + "billingCity": "San Francisco", + "website": "https://example.com" + } + }, + "metadata": { + "salesforce.account.id": "001XX000003DHPh", + "salesforce.account.owner_id": "005XX000001SvogAAC", + "salesforce.account.industry": "Technology", + "salesforce.account.billing_city": "San Francisco", + "salesforce.account.website": "https://example.com" + } +} +``` + +### Salesforce Opportunity Resource + +```json +{ + "version": "ctrlplane.dev/crm/opportunity/v1", + "kind": "SalesforceOpportunity", + "name": "Opportunity Name", + "identifier": "006XX000003DHPh", + "config": { + "name": "Opportunity Name", + "amount": "50000.00", + "stage": "Qualification", + "id": "006XX000003DHPh", + "salesforceOpportunity": { + "recordId": "006XX000003DHPh", + "closeDate": "2024-12-31T00:00:00Z", + "accountId": "001XX000003DHPh", + "probability": "10" + } + }, + "metadata": { + "salesforce.opportunity.id": "006XX000003DHPh", + "salesforce.opportunity.account_id": "001XX000003DHPh", + "salesforce.opportunity.stage": "Qualification", + "salesforce.opportunity.amount": "50000.00", + "salesforce.opportunity.probability": "10", + "salesforce.opportunity.close_date": "2024-12-31" + } +} +``` + +## Implementation Details + +This integration uses the [go-salesforce](https://github.com/k-capehart/go-salesforce) library for OAuth2 authentication and SOQL queries. + +### Features + +- **Dynamic Field Discovery**: Automatically discovers and fetches all available fields (standard and custom) from Salesforce objects +- **Custom Field Mappings**: Map any Salesforce field to Ctrlplane metadata using command-line flags +- **Flexible Filtering**: Use SOQL WHERE clauses with the `--where` flag to filter records +- **Flexible Transformations**: Use reflection-based utilities to handle field mappings dynamically +- **Extensible Architecture**: Shared utilities in `common/util.go` make it easy to add support for new Salesforce objects +- **Automatic Pagination**: Fetches all records by default, with support for limiting the number of records +- **Smart Field Capture**: Automatically captures any custom Salesforce fields not defined in the struct +- **Optional Field Listing**: Use `--list-all-fields` to see all available fields in the logs + +### Currently Syncs + +- **Accounts**: Complete account information including: + - Basic fields (name, industry, website, phone) + - Address information (billing and shipping) + - Hierarchy (parent/child relationships) + - Custom fields (any field ending in `__c`) + +- **Opportunities**: Deal information including: + - Basic fields (name, amount, stage, close date) + - Relationships (account associations) + - Probability and forecast data + - Custom fields (any field ending in `__c`) + +### Pagination + +By default, the sync will retrieve all records from Salesforce using pagination: +- Records are fetched in batches of 2000 (Salesforce's default) +- Uses ID-based pagination to handle large datasets (avoids Salesforce's OFFSET limitation) +- Use the `--limit` flag to restrict the number of records synced +- Records are ordered by ID for consistent pagination + +### Shared Utilities + +The `common/util.go` file provides reusable functions for all Salesforce object syncs: +- `QuerySalesforceObject`: Generic function to query any Salesforce object with pagination support +- `UnmarshalWithCustomFields`: Captures any fields from Salesforce that aren't defined in the Go struct +- `GetKnownFieldsFromStruct`: Automatically extracts field names from struct tags +- `ParseMappings`: Handles custom metadata field mappings + +These utilities make it easy to add support for new Salesforce objects (Leads, Contacts, etc.) with minimal code duplication. \ No newline at end of file diff --git a/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go new file mode 100644 index 0000000..6f20c44 --- /dev/null +++ b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go @@ -0,0 +1,286 @@ +package accounts + +import ( + "context" + "reflect" + + "github.com/MakeNowJust/heredoc/v2" + "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/common" + "github.com/ctrlplanedev/cli/internal/api" + "github.com/k-capehart/go-salesforce/v2" + "github.com/spf13/cobra" +) + +type Account struct { + ID string `json:"Id"` // this is the globally unique identifier for the account + IsDeleted bool `json:"IsDeleted"` + MasterRecordId string `json:"MasterRecordId"` + Name string `json:"Name"` + Type string `json:"Type"` + ParentId string `json:"ParentId"` + BillingStreet string `json:"BillingStreet"` + BillingCity string `json:"BillingCity"` + BillingState string `json:"BillingState"` + BillingPostalCode string `json:"BillingPostalCode"` + BillingCountry string `json:"BillingCountry"` + BillingLatitude float64 `json:"BillingLatitude"` + BillingLongitude float64 `json:"BillingLongitude"` + BillingGeocodeAccuracy string `json:"BillingGeocodeAccuracy"` + BillingAddress interface{} `json:"BillingAddress"` + ShippingStreet string `json:"ShippingStreet"` + ShippingCity string `json:"ShippingCity"` + ShippingState string `json:"ShippingState"` + ShippingPostalCode string `json:"ShippingPostalCode"` + ShippingCountry string `json:"ShippingCountry"` + ShippingLatitude float64 `json:"ShippingLatitude"` + ShippingLongitude float64 `json:"ShippingLongitude"` + ShippingGeocodeAccuracy string `json:"ShippingGeocodeAccuracy"` + ShippingAddress interface{} `json:"ShippingAddress"` + Phone string `json:"Phone"` + Website string `json:"Website"` + PhotoUrl string `json:"PhotoUrl"` + Industry string `json:"Industry"` + NumberOfEmployees int `json:"NumberOfEmployees"` + Description string `json:"Description"` + OwnerId string `json:"OwnerId"` + CreatedDate string `json:"CreatedDate"` + CreatedById string `json:"CreatedById"` + LastModifiedDate string `json:"LastModifiedDate"` + LastModifiedById string `json:"LastModifiedById"` + SystemModstamp string `json:"SystemModstamp"` + LastActivityDate string `json:"LastActivityDate"` + LastViewedDate string `json:"LastViewedDate"` + LastReferencedDate string `json:"LastReferencedDate"` + Jigsaw string `json:"Jigsaw"` + JigsawCompanyId string `json:"JigsawCompanyId"` + AccountSource string `json:"AccountSource"` + SicDesc string `json:"SicDesc"` + IsPriorityRecord bool `json:"IsPriorityRecord"` + + // CustomFields holds any additional fields not defined in the struct + // This allows handling of custom Salesforce fields like Tier__c with + // the --metadata flag. + CustomFields map[string]interface{} `json:"-"` +} + +// GetCustomFields implements the DynamicFieldHolder interface +func (a Account) GetCustomFields() map[string]interface{} { + return a.CustomFields +} + +func (a *Account) UnmarshalJSON(data []byte) error { + type Alias Account + aux := &struct { + *Alias + }{ + Alias: (*Alias)(a), + } + + knownFields := common.GetKnownFieldsFromStruct(reflect.TypeOf(Account{})) + + customFields, err := common.UnmarshalWithCustomFields(data, aux, knownFields) + if err != nil { + return err + } + + a.CustomFields = customFields + + return nil +} + +func NewSalesforceAccountsCmd() *cobra.Command { + var name string + var domain string + var consumerKey string + var consumerSecret string + var metadataMappings []string + var limit int + var listAllFields bool + var whereClause string + + cmd := &cobra.Command{ + Use: "accounts", + Short: "Sync Salesforce accounts into Ctrlplane", + Example: heredoc.Doc(` + # Make sure Salesforce credentials are configured via environment variables + + # Sync all Salesforce accounts + $ ctrlc sync salesforce accounts + + # Sync accounts with a specific filter + $ ctrlc sync salesforce accounts --where="Customer_Health__c != null" + + # Sync accounts and list all available fields in logs + $ ctrlc sync salesforce accounts --list-all-fields + + # Sync accounts with custom provider name + $ ctrlc sync salesforce accounts --provider my-salesforce + + # Sync with custom metadata mappings + $ ctrlc sync salesforce accounts \ + --metadata="account/id=Id" \ + --metadata="account/owner-id=OwnerId" \ + --metadata="account/tier=Tier__c" \ + --metadata="account/region=Region__c" \ + --metadata="account/annual-revenue=Annual_Revenue__c" \ + --metadata="account/health=Customer_Health__c" + + # Sync with a limit on number of records + $ ctrlc sync salesforce accounts --limit 500 + + # Combine filters with metadata mappings + $ ctrlc sync salesforce accounts \ + --where="Type = 'Customer' AND AnnualRevenue > 1000000" \ + --metadata="account/revenue=AnnualRevenue" + `), + PreRunE: common.ValidateFlags(&domain, &consumerKey, &consumerSecret), + RunE: runSync(&name, &domain, &consumerKey, &consumerSecret, &metadataMappings, &limit, &listAllFields, &whereClause), + } + + // Add command flags + cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") + cmd.Flags().StringVar(&domain, "domain", "", "Salesforce domain (e.g., https://my-domain.my.salesforce.com)") + cmd.Flags().StringVar(&consumerKey, "consumer-key", "", "Salesforce consumer key") + cmd.Flags().StringVar(&consumerSecret, "consumer-secret", "", "Salesforce consumer secret") + cmd.Flags().StringArrayVar(&metadataMappings, "metadata", []string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") + cmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of records to sync (0 = no limit)") + cmd.Flags().BoolVar(&listAllFields, "list-all-fields", false, "List all available Salesforce fields in the logs") + cmd.Flags().StringVar(&whereClause, "where", "", "SOQL WHERE clause to filter records (e.g., \"Customer_Health__c != null\")") + + return cmd +} + +// runSync contains the main sync logic +func runSync(name, domain, consumerKey, consumerSecret *string, metadataMappings *[]string, limit *int, listAllFields *bool, whereClause *string) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + log.Info("Syncing Salesforce accounts into Ctrlplane", "domain", *domain) + + ctx := context.Background() + + sf, err := common.InitSalesforceClient(*domain, *consumerKey, *consumerSecret) + if err != nil { + return err + } + + resources, err := processAccounts(ctx, sf, *metadataMappings, *limit, *listAllFields, *whereClause) + if err != nil { + return err + } + + if *name == "" { + *name = "salesforce-accounts" + } + + return common.UpsertToCtrlplane(ctx, resources, *name) + } +} + +// processAccounts queries and transforms accounts +func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMappings []string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { + + accounts, err := queryAccounts(ctx, sf, limit, listAllFields, metadataMappings, whereClause) + if err != nil { + return nil, err + } + + log.Info("Found Salesforce accounts", "count", len(accounts)) + + resources := []api.CreateResource{} + for _, account := range accounts { + resource := transformAccountToResource(account, metadataMappings) + resources = append(resources, resource) + } + + return resources, nil +} + +func queryAccounts(ctx context.Context, sf *salesforce.Salesforce, limit int, listAllFields bool, metadataMappings []string, whereClause string) ([]Account, error) { + additionalFields := common.ExtractFieldsFromMetadataMappings(metadataMappings) + + var accounts []Account + err := common.QuerySalesforceObject(ctx, sf, "Account", limit, listAllFields, &accounts, additionalFields, whereClause) + if err != nil { + return nil, err + } + return accounts, nil +} + +func transformAccountToResource(account Account, metadataMappings []string) api.CreateResource { + defaultMetadataMappings := map[string]string{ + "ctrlplane/external-id": "Id", + "account/id": "Id", + "account/owner-id": "OwnerId", + "account/industry": "Industry", + "account/billing-city": "BillingCity", + "account/billing-state": "BillingState", + "account/billing-country": "BillingCountry", + "account/website": "Website", + "account/phone": "Phone", + "account/type": "Type", + "account/source": "AccountSource", + "account/shipping-city": "ShippingCity", + "account/parent-id": "ParentId", + "account/employees": "NumberOfEmployees", + "account/region": "Region__c", + "account/annual-revenue": "Annual_Revenue__c", + "account/tier": "Tier__c", + "account/health": "Customer_Health__c", + } + + // Parse metadata mappings using common utility + metadata := common.ParseMappings(account, metadataMappings, defaultMetadataMappings) + + // Build base config with common fields + config := map[string]interface{}{ + // Common cross-provider fields + "name": account.Name, + "industry": account.Industry, + "id": account.ID, + "type": account.Type, + "phone": account.Phone, + "website": account.Website, + + // Salesforce-specific implementation details + "salesforceAccount": map[string]interface{}{ + "recordId": account.ID, + "ownerId": account.OwnerId, + "parentId": account.ParentId, + "type": account.Type, + "accountSource": account.AccountSource, + "numberOfEmployees": account.NumberOfEmployees, + "description": account.Description, + "billingAddress": map[string]interface{}{ + "street": account.BillingStreet, + "city": account.BillingCity, + "state": account.BillingState, + "postalCode": account.BillingPostalCode, + "country": account.BillingCountry, + "latitude": account.BillingLatitude, + "longitude": account.BillingLongitude, + }, + "shippingAddress": map[string]interface{}{ + "street": account.ShippingStreet, + "city": account.ShippingCity, + "state": account.ShippingState, + "postalCode": account.ShippingPostalCode, + "country": account.ShippingCountry, + "latitude": account.ShippingLatitude, + "longitude": account.ShippingLongitude, + }, + "createdDate": account.CreatedDate, + "lastModifiedDate": account.LastModifiedDate, + "isDeleted": account.IsDeleted, + "photoUrl": account.PhotoUrl, + }, + } + + return api.CreateResource{ + Version: "ctrlplane.dev/crm/account/v1", + Kind: "SalesforceAccount", + Name: account.Name, + Identifier: account.ID, + Config: config, + Metadata: metadata, + } +} diff --git a/cmd/ctrlc/root/sync/salesforce/common/client.go b/cmd/ctrlc/root/sync/salesforce/common/client.go new file mode 100644 index 0000000..804830d --- /dev/null +++ b/cmd/ctrlc/root/sync/salesforce/common/client.go @@ -0,0 +1,41 @@ +package common + +import ( + "fmt" + "os" + + "github.com/k-capehart/go-salesforce/v2" + "github.com/spf13/cobra" +) + +func InitSalesforceClient(domain, consumerKey, consumerSecret string) (*salesforce.Salesforce, error) { + sf, err := salesforce.Init(salesforce.Creds{ + Domain: domain, + ConsumerKey: consumerKey, + ConsumerSecret: consumerSecret, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize Salesforce client: %w", err) + } + return sf, nil +} + +func ValidateFlags(domain, consumerKey, consumerSecret *string) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + if *domain == "" { + *domain = os.Getenv("SALESFORCE_DOMAIN") + } + if *consumerKey == "" { + *consumerKey = os.Getenv("SALESFORCE_CONSUMER_KEY") + } + if *consumerSecret == "" { + *consumerSecret = os.Getenv("SALESFORCE_CONSUMER_SECRET") + } + + if *domain == "" || *consumerKey == "" || *consumerSecret == "" { + return fmt.Errorf("Salesforce credentials are required. Set SALESFORCE_DOMAIN, SALESFORCE_CONSUMER_KEY, and SALESFORCE_CONSUMER_SECRET environment variables or use flags") + } + + return nil + } +} diff --git a/cmd/ctrlc/root/sync/salesforce/common/util.go b/cmd/ctrlc/root/sync/salesforce/common/util.go new file mode 100644 index 0000000..1cc2bc6 --- /dev/null +++ b/cmd/ctrlc/root/sync/salesforce/common/util.go @@ -0,0 +1,430 @@ +package common + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/url" + "reflect" + "strings" + + "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/internal/api" + "github.com/k-capehart/go-salesforce/v2" + "github.com/spf13/viper" +) + +// DynamicFieldHolder interface for structs that can hold custom fields +type DynamicFieldHolder interface { + GetCustomFields() map[string]interface{} +} + +// ExtractFieldsFromMetadataMappings extracts Salesforce field names from metadata mappings +func ExtractFieldsFromMetadataMappings(metadataMappings []string) []string { + fieldMap := make(map[string]bool) + + for _, mapping := range metadataMappings { + parts := strings.SplitN(mapping, "=", 2) + if len(parts) == 2 { + salesforceField := strings.TrimSpace(parts[1]) + if salesforceField != "" { + fieldMap[salesforceField] = true + } + } + } + + fields := make([]string, 0, len(fieldMap)) + for field := range fieldMap { + fields = append(fields, field) + } + + return fields +} + +// ParseMappings applies custom field mappings to extract string values for metadata +func ParseMappings(data interface{}, mappings []string, defaultMappings map[string]string) map[string]string { + result := map[string]string{} + + dataValue := reflect.ValueOf(data) + dataType := reflect.TypeOf(data) + + if dataValue.Kind() == reflect.Ptr { + dataValue = dataValue.Elem() + dataType = dataType.Elem() + } + + // Process custom mappings (format: ctrlplane/key=SalesforceField) + for _, mapping := range mappings { + parts := strings.SplitN(mapping, "=", 2) + if len(parts) != 2 { + log.Warn("Invalid mapping format, skipping", "mapping", mapping) + continue + } + + ctrlplaneKey, sfField := parts[0], parts[1] + + found := false + for i := 0; i < dataType.NumField(); i++ { + field := dataType.Field(i) + jsonTag := field.Tag.Get("json") + + if jsonTag == sfField { + fieldValue := dataValue.Field(i) + + strValue := fieldToString(fieldValue) + + if strValue != "" { + result[ctrlplaneKey] = strValue + } + found = true + break + } + } + + // If not found in struct fields, check CustomFields if the struct implements DynamicFieldHolder + if !found { + if holder, ok := data.(DynamicFieldHolder); ok { + customFields := holder.GetCustomFields() + if customFields != nil { + if value, exists := customFields[sfField]; exists { + strValue := fmt.Sprintf("%v", value) + if strValue != "" && strValue != "" { + result[ctrlplaneKey] = strValue + } + } + } + } + } + } + + // Add default mappings if they haven't been explicitly mapped + existingKeys := make(map[string]bool) + for key := range result { + existingKeys[key] = true + } + + for defaultKey, sfField := range defaultMappings { + if !existingKeys[defaultKey] { + // Find and add the default field + for i := 0; i < dataType.NumField(); i++ { + field := dataType.Field(i) + if field.Tag.Get("json") == sfField { + fieldValue := dataValue.Field(i) + strValue := fieldToString(fieldValue) + + if strValue != "" { + result[defaultKey] = strValue + } + break + } + } + } + } + + return result +} + +// fieldToString converts a reflect.Value to string for metadata +func fieldToString(fieldValue reflect.Value) string { + switch fieldValue.Kind() { + case reflect.String: + return fieldValue.String() + case reflect.Int, reflect.Int64: + if val := fieldValue.Int(); val != 0 { + return fmt.Sprintf("%d", val) + } + case reflect.Float64, reflect.Float32: + if val := fieldValue.Float(); val != 0 { + return fmt.Sprintf("%g", val) // %g removes trailing zeros + } + case reflect.Bool: + if fieldValue.Bool() { + return "true" + } + return "false" + default: + // For complex types, try to marshal to JSON + if fieldValue.IsValid() && fieldValue.CanInterface() { + if bytes, err := json.Marshal(fieldValue.Interface()); err == nil { + return string(bytes) + } + } + } + + return "" +} + +// QuerySalesforceObject performs a generic query on any Salesforce object with pagination support +func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objectName string, limit int, listAllFields bool, target interface{}, additionalFields []string, whereClause string) error { + targetValue := reflect.ValueOf(target).Elem() + if targetValue.Kind() != reflect.Slice { + return fmt.Errorf("target must be a pointer to a slice") + } + elementType := targetValue.Type().Elem() + + // Get field names from the struct + fieldNames := []string{} + for i := 0; i < elementType.NumField(); i++ { + field := elementType.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag != "" && jsonTag != "-" { + fieldName := strings.Split(jsonTag, ",")[0] + if fieldName != "" { + fieldNames = append(fieldNames, fieldName) + } + } + } + + // Include additional fields passed in (e.g., from metadata mappings) + for _, field := range additionalFields { + found := false + for _, existing := range fieldNames { + if existing == field { + found = true + break + } + } + if !found { + fieldNames = append(fieldNames, field) + } + } + + // If listAllFields is true, describe the object to show what's available + if listAllFields { + describeResp, err := sf.DoRequest("GET", fmt.Sprintf("/sobjects/%s/describe", objectName), nil) + if err != nil { + return fmt.Errorf("failed to describe %s object: %w", objectName, err) + } + defer describeResp.Body.Close() + + var describeResult map[string]interface{} + if err := json.NewDecoder(describeResp.Body).Decode(&describeResult); err != nil { + return fmt.Errorf("failed to decode describe response: %w", err) + } + + // Extract all available field names for logging + fields, ok := describeResult["fields"].([]interface{}) + if !ok { + return fmt.Errorf("unexpected describe response format") + } + + allFieldNames := []string{} + for _, field := range fields { + fieldMap, ok := field.(map[string]interface{}) + if !ok { + continue + } + if name, ok := fieldMap["name"].(string); ok { + allFieldNames = append(allFieldNames, name) + } + } + log.Info("Available fields", "object", objectName, "count", len(allFieldNames), "fields", allFieldNames) + } + + // Build query with pagination support + totalRetrieved := 0 + lastId := "" + batchSize := 2000 + + // Query in batches using ID-based pagination to avoid OFFSET limits + for { + fieldsClause := strings.Join(fieldNames, ", ") + baseQuery := fmt.Sprintf("SELECT %s FROM %s", fieldsClause, objectName) + + paginatedQuery := baseQuery + whereClauses := []string{} + + if whereClause != "" { + whereClauses = append(whereClauses, whereClause) + } + + if lastId != "" { + whereClauses = append(whereClauses, fmt.Sprintf("Id > '%s'", lastId)) + } + + if len(whereClauses) > 0 { + paginatedQuery += " WHERE " + strings.Join(whereClauses, " AND ") + } + paginatedQuery += " ORDER BY Id" + + if limit > 0 && limit-totalRetrieved < batchSize { + paginatedQuery += fmt.Sprintf(" LIMIT %d", limit-totalRetrieved) + } else { + paginatedQuery += fmt.Sprintf(" LIMIT %d", batchSize) + } + + encodedQuery := url.QueryEscape(paginatedQuery) + queryURL := fmt.Sprintf("/query?q=%s", encodedQuery) + + queryResp, err := sf.DoRequest("GET", queryURL, nil) + if err != nil { + return fmt.Errorf("failed to query %s: %w", objectName, err) + } + defer queryResp.Body.Close() + + body, err := io.ReadAll(queryResp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + var queryResult struct { + TotalSize int `json:"totalSize"` + Done bool `json:"done"` + Records json.RawMessage `json:"records"` + NextRecordsUrl string `json:"nextRecordsUrl"` + } + + if err := json.Unmarshal(body, &queryResult); err != nil { + return fmt.Errorf("failed to unmarshal query response: %w", err) + } + + batchSlice := reflect.New(targetValue.Type()).Elem() + + // Unmarshal records into the batch slice - this will trigger our custom UnmarshalJSON + if err := json.Unmarshal(queryResult.Records, batchSlice.Addr().Interface()); err != nil { + return fmt.Errorf("failed to unmarshal records: %w", err) + } + + if batchSlice.Len() == 0 { + break + } + + for i := 0; i < batchSlice.Len(); i++ { + targetValue.Set(reflect.Append(targetValue, batchSlice.Index(i))) + } + + recordCount := batchSlice.Len() + totalRetrieved += recordCount + + if recordCount > 0 { + lastRecord := batchSlice.Index(recordCount - 1) + if lastRecord.Kind() == reflect.Struct { + idField := lastRecord.FieldByName("ID") + if !idField.IsValid() { + idField = lastRecord.FieldByName("Id") + } + if idField.IsValid() && idField.Kind() == reflect.String { + lastId = idField.String() + } + } + } + + log.Debug("Retrieved batch", "object", objectName, "batch_size", recordCount, "total", totalRetrieved) + + if limit > 0 && totalRetrieved >= limit { + break + } + + if recordCount == 0 { + break + } + } + + if limit > 0 && targetValue.Len() > limit { + targetValue.Set(targetValue.Slice(0, limit)) + } + + return nil +} + +// ExtractCustomFields extracts fields from a JSON response that aren't in the struct +func ExtractCustomFields(data []byte, knownFields map[string]bool) (map[string]interface{}, error) { + var allFields map[string]interface{} + if err := json.Unmarshal(data, &allFields); err != nil { + return nil, err + } + + customFields := make(map[string]interface{}) + for key, value := range allFields { + if !knownFields[key] { + customFields[key] = value + } + } + + return customFields, nil +} + +// GetKnownFieldsFromStruct extracts all JSON field names from a struct's tags +func GetKnownFieldsFromStruct(structType reflect.Type) map[string]bool { + knownFields := make(map[string]bool) + + if structType.Kind() == reflect.Ptr { + structType = structType.Elem() + } + + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + jsonTag := field.Tag.Get("json") + + if jsonTag == "" || jsonTag == "-" { + continue + } + + fieldName := strings.Split(jsonTag, ",")[0] + if fieldName != "" { + knownFields[fieldName] = true + } + } + + return knownFields +} + +// UpsertToCtrlplane creates or updates a resource provider and sets its resources +func UpsertToCtrlplane(ctx context.Context, resources []api.CreateResource, providerName string) error { + apiURL := viper.GetString("url") + apiKey := viper.GetString("api-key") + workspaceId := viper.GetString("workspace") + + ctrlplaneClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + + providerResp, err := ctrlplaneClient.UpsertResourceProviderWithResponse(ctx, workspaceId, providerName) + if err != nil { + return fmt.Errorf("failed to upsert resource provider: %w", err) + } + + if providerResp.JSON200 == nil { + return fmt.Errorf("failed to upsert resource provider: %s", providerResp.Body) + } + + providerId := providerResp.JSON200.Id + log.Info("Upserting resources", "provider", providerName, "count", len(resources)) + + setResp, err := ctrlplaneClient.SetResourceProvidersResourcesWithResponse(ctx, providerId, api.SetResourceProvidersResourcesJSONRequestBody{ + Resources: resources, + }) + if err != nil { + return fmt.Errorf("failed to set resources: %w", err) + } + + if setResp.JSON200 == nil { + return fmt.Errorf("failed to set resources: %s", setResp.Body) + } + + log.Info("Successfully synced resources", "count", len(resources)) + return nil +} + +// UnmarshalWithCustomFields unmarshals JSON data into the target struct and returns any unknown fields +func UnmarshalWithCustomFields(data []byte, target interface{}, knownFields map[string]bool) (map[string]interface{}, error) { + if err := json.Unmarshal(data, target); err != nil { + return nil, err + } + + var allFields map[string]interface{} + if err := json.Unmarshal(data, &allFields); err != nil { + return nil, err + } + + customFields := make(map[string]interface{}) + for fieldName, value := range allFields { + if !knownFields[fieldName] { + customFields[fieldName] = value + } + } + + return customFields, nil +} diff --git a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go new file mode 100644 index 0000000..0b427a4 --- /dev/null +++ b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go @@ -0,0 +1,273 @@ +package opportunities + +import ( + "context" + "reflect" + "strconv" + "time" + + "github.com/MakeNowJust/heredoc/v2" + "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/common" + "github.com/ctrlplanedev/cli/internal/api" + "github.com/k-capehart/go-salesforce/v2" + "github.com/spf13/cobra" +) + +type Opportunity struct { + ID string `json:"Id"` + Name string `json:"Name"` + Amount float64 `json:"Amount"` + StageName string `json:"StageName"` + CloseDate string `json:"CloseDate"` // Salesforce returns dates as strings + AccountID string `json:"AccountId"` + Probability float64 `json:"Probability"` + IsDeleted bool `json:"IsDeleted"` + Description string `json:"Description"` + Type string `json:"Type"` + NextStep string `json:"NextStep"` + LeadSource string `json:"LeadSource"` + IsClosed bool `json:"IsClosed"` + IsWon bool `json:"IsWon"` + ForecastCategory string `json:"ForecastCategory"` + ForecastCategoryName string `json:"ForecastCategoryName"` + CampaignID string `json:"CampaignId"` + HasOpportunityLineItem bool `json:"HasOpportunityLineItem"` + Pricebook2ID string `json:"Pricebook2Id"` + OwnerID string `json:"OwnerId"` + Territory2ID string `json:"Territory2Id"` + IsExcludedFromTerritory2Filter bool `json:"IsExcludedFromTerritory2Filter"` + CreatedDate string `json:"CreatedDate"` + CreatedById string `json:"CreatedById"` + LastModifiedDate string `json:"LastModifiedDate"` + LastModifiedById string `json:"LastModifiedById"` + SystemModstamp string `json:"SystemModstamp"` + LastActivityDate string `json:"LastActivityDate"` + PushCount int `json:"PushCount"` + LastStageChangeDate string `json:"LastStageChangeDate"` + ContactId string `json:"ContactId"` + LastViewedDate string `json:"LastViewedDate"` + LastReferencedDate string `json:"LastReferencedDate"` + SyncedQuoteId string `json:"SyncedQuoteId"` + ContractId string `json:"ContractId"` + HasOpenActivity bool `json:"HasOpenActivity"` + HasOverdueTask bool `json:"HasOverdueTask"` + LastAmountChangedHistoryId string `json:"LastAmountChangedHistoryId"` + LastCloseDateChangedHistoryId string `json:"LastCloseDateChangedHistoryId"` + + // CustomFields holds any additional fields not defined in the struct + CustomFields map[string]interface{} `json:"-"` +} + +// GetCustomFields implements the DynamicFieldHolder interface +func (o Opportunity) GetCustomFields() map[string]interface{} { + return o.CustomFields +} + +// UnmarshalJSON implements custom unmarshalling to capture unknown fields +func (o *Opportunity) UnmarshalJSON(data []byte) error { + type Alias Opportunity + aux := &struct { + *Alias + }{ + Alias: (*Alias)(o), + } + + knownFields := common.GetKnownFieldsFromStruct(reflect.TypeOf(Opportunity{})) + + customFields, err := common.UnmarshalWithCustomFields(data, aux, knownFields) + if err != nil { + return err + } + + o.CustomFields = customFields + + return nil +} + +func NewSalesforceOpportunitiesCmd() *cobra.Command { + var name string + var domain string + var consumerKey string + var consumerSecret string + var metadataMappings []string + var limit int + var listAllFields bool + var whereClause string + + cmd := &cobra.Command{ + Use: "opportunities", + Short: "Sync Salesforce opportunities into Ctrlplane", + Example: heredoc.Doc(` + # Sync all Salesforce opportunities + $ ctrlc sync salesforce opportunities + + # Sync opportunities with a specific filter + $ ctrlc sync salesforce opportunities --where="IsWon = true" + + # Sync opportunities with custom provider name + $ ctrlc sync salesforce opportunities --provider my-salesforce-opportunities + + # Sync with custom metadata mappings + $ ctrlc sync salesforce opportunities \ + --metadata="opportunity/stage=StageName" \ + --metadata="opportunity/amount=Amount" \ + --metadata="opportunity/close-date=CloseDate" + + # Sync with a limit on number of records + $ ctrlc sync salesforce opportunities --limit 100 + + # Combine filters with metadata mappings + $ ctrlc sync salesforce opportunities \ + --where="Amount > 50000 AND StageName != 'Closed Lost'" \ + --metadata="opportunity/probability=Probability" + `), + PreRunE: common.ValidateFlags(&domain, &consumerKey, &consumerSecret), + RunE: runSync(&name, &domain, &consumerKey, &consumerSecret, &metadataMappings, &limit, &listAllFields, &whereClause), + } + + // Add command flags + cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") + cmd.Flags().StringVar(&domain, "domain", "", "Salesforce domain (e.g., https://my-domain.my.salesforce.com)") + cmd.Flags().StringVar(&consumerKey, "consumer-key", "", "Salesforce consumer key") + cmd.Flags().StringVar(&consumerSecret, "consumer-secret", "", "Salesforce consumer secret") + cmd.Flags().StringArrayVar(&metadataMappings, "metadata", []string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") + cmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of records to sync (0 = no limit)") + cmd.Flags().BoolVar(&listAllFields, "list-all-fields", false, "List all available Salesforce fields in the logs") + cmd.Flags().StringVar(&whereClause, "where", "", "SOQL WHERE clause to filter records (e.g., \"IsClosed = false\")") + + return cmd +} + +// runSync contains the main sync logic +func runSync(name, domain, consumerKey, consumerSecret *string, metadataMappings *[]string, limit *int, listAllFields *bool, whereClause *string) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + log.Info("Syncing Salesforce opportunities into Ctrlplane", "domain", *domain) + + ctx := context.Background() + + sf, err := common.InitSalesforceClient(*domain, *consumerKey, *consumerSecret) + if err != nil { + return err + } + + resources, err := processOpportunities(ctx, sf, *metadataMappings, *limit, *listAllFields, *whereClause) + if err != nil { + return err + } + + if *name == "" { + *name = "salesforce-opportunities" + } + + return common.UpsertToCtrlplane(ctx, resources, *name) + } +} + +// processOpportunities queries and transforms opportunities +func processOpportunities(ctx context.Context, sf *salesforce.Salesforce, metadataMappings []string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { + opportunities, err := queryOpportunities(ctx, sf, limit, listAllFields, metadataMappings, whereClause) + if err != nil { + return nil, err + } + + log.Info("Found Salesforce opportunities", "count", len(opportunities)) + + // Transform to ctrlplane resources + resources := []api.CreateResource{} + for _, opp := range opportunities { + resource := transformOpportunityToResource(opp, metadataMappings) + resources = append(resources, resource) + } + + return resources, nil +} + +func queryOpportunities(ctx context.Context, sf *salesforce.Salesforce, limit int, listAllFields bool, metadataMappings []string, whereClause string) ([]Opportunity, error) { + // Extract Salesforce field names from metadata mappings + additionalFields := common.ExtractFieldsFromMetadataMappings(metadataMappings) + + var opportunities []Opportunity + err := common.QuerySalesforceObject(ctx, sf, "Opportunity", limit, listAllFields, &opportunities, additionalFields, whereClause) + if err != nil { + return nil, err + } + return opportunities, nil +} + +func transformOpportunityToResource(opportunity Opportunity, metadataMappings []string) api.CreateResource { + // Define default metadata mappings for opportunities + defaultMetadataMappings := map[string]string{ + "opportunity/id": "Id", + "ctrlplane/external-id": "Id", + "opportunity/account-id": "AccountId", + "opportunity/stage": "StageName", + "opportunity/amount": "Amount", + "opportunity/probability": "Probability", + "opportunity/close-date": "CloseDate", + "opportunity/name": "Name", + "opportunity/type": "Type", + "opportunity/owner-id": "OwnerId", + "opportunity/is-closed": "IsClosed", + "opportunity/is-won": "IsWon", + "opportunity/lead-source": "LeadSource", + "opportunity/forecast-category": "ForecastCategory", + "opportunity/contact-id": "ContactId", + "opportunity/campaign-id": "CampaignId", + "opportunity/created-date": "CreatedDate", + "opportunity/last-modified": "LastModifiedDate", + } + + // Parse metadata mappings using common utility + metadata := common.ParseMappings(opportunity, metadataMappings, defaultMetadataMappings) + + var closeDateFormatted string + if opportunity.CloseDate != "" { + if t, err := time.Parse("2006-01-02", opportunity.CloseDate); err == nil { + closeDateFormatted = t.Format(time.RFC3339) + } else { + closeDateFormatted = opportunity.CloseDate + } + } + + // Build base config with common fields + config := map[string]interface{}{ + // Common cross-provider fields + "name": opportunity.Name, + "amount": strconv.FormatFloat(opportunity.Amount, 'f', 2, 64), + "stage": opportunity.StageName, + "id": opportunity.ID, + + // Salesforce-specific implementation details + "salesforceOpportunity": map[string]interface{}{ + "recordId": opportunity.ID, + "closeDate": closeDateFormatted, + "accountId": opportunity.AccountID, + "probability": strconv.FormatFloat(opportunity.Probability, 'f', 2, 64), + "type": opportunity.Type, + "description": opportunity.Description, + "nextStep": opportunity.NextStep, + "leadSource": opportunity.LeadSource, + "isClosed": opportunity.IsClosed, + "isWon": opportunity.IsWon, + "forecastCategory": opportunity.ForecastCategory, + "ownerId": opportunity.OwnerID, + "contactId": opportunity.ContactId, + "campaignId": opportunity.CampaignID, + "hasLineItems": opportunity.HasOpportunityLineItem, + "createdDate": opportunity.CreatedDate, + "lastModifiedDate": opportunity.LastModifiedDate, + "pushCount": opportunity.PushCount, + "lastStageChangeDate": opportunity.LastStageChangeDate, + }, + } + + return api.CreateResource{ + Version: "ctrlplane.dev/crm/opportunity/v1", + Kind: "SalesforceOpportunity", + Name: opportunity.Name, + Identifier: opportunity.ID, + Config: config, + Metadata: metadata, + } +} diff --git a/cmd/ctrlc/root/sync/salesforce/salesforce.go b/cmd/ctrlc/root/sync/salesforce/salesforce.go new file mode 100644 index 0000000..ecba6e3 --- /dev/null +++ b/cmd/ctrlc/root/sync/salesforce/salesforce.go @@ -0,0 +1,25 @@ +package salesforce + +import ( + "github.com/MakeNowJust/heredoc" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/accounts" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/opportunities" + "github.com/spf13/cobra" +) + +func NewSalesforceCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "salesforce", + Short: "Sync Salesforce resources into Ctrlplane", + Example: heredoc.Doc(` + # Sync all Salesforce objects + $ ctrlc sync salesforce accounts + $ ctrlc sync salesforce opportunities + `), + } + + cmd.AddCommand(accounts.NewSalesforceAccountsCmd()) + cmd.AddCommand(opportunities.NewSalesforceOpportunitiesCmd()) + + return cmd +} diff --git a/cmd/ctrlc/root/sync/sync.go b/cmd/ctrlc/root/sync/sync.go index 6c430ae..82d5482 100644 --- a/cmd/ctrlc/root/sync/sync.go +++ b/cmd/ctrlc/root/sync/sync.go @@ -8,6 +8,7 @@ import ( "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/github" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/google" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/kubernetes" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/tailscale" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/terraform" "github.com/ctrlplanedev/cli/internal/cliutil" @@ -38,6 +39,7 @@ func NewSyncCmd() *cobra.Command { cmd.AddCommand(kubernetes.NewSyncKubernetesCmd()) cmd.AddCommand(kubernetes.NewSyncVclusterCmd()) cmd.AddCommand(github.NewSyncGitHubCmd()) + cmd.AddCommand(salesforce.NewSalesforceCmd()) return cmd } diff --git a/go.mod b/go.mod index db29724..d166415 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0 + github.com/MakeNowJust/heredoc v1.0.0 github.com/MakeNowJust/heredoc/v2 v2.0.1 github.com/Masterminds/semver v1.5.0 github.com/avast/retry-go v3.0.0+incompatible @@ -26,6 +27,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-tfe v1.73.1 + github.com/k-capehart/go-salesforce/v2 v2.5.2 github.com/loft-sh/vcluster v0.25.0 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/term v0.5.0 @@ -38,7 +40,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.32.1 - k8s.io/apimachinery v0.32.1 + k8s.io/apimachinery v0.33.2 k8s.io/client-go v0.32.1 ) @@ -51,7 +53,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect - github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect @@ -86,6 +87,7 @@ require ( github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/forcedotcom/go-soql v0.0.0-20220705175410-00f698360bee // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/getkin/kin-openapi v0.127.0 // indirect @@ -96,17 +98,17 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-viper/mapstructure/v2 v2.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.22.0 // indirect - github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect + github.com/google/gnostic-models v0.6.9 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-github/v53 v53.2.1-0.20230815134205-bb00f570d301 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect @@ -126,6 +128,7 @@ require ( github.com/jonboulle/clockwork v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/jszwec/csvutil v1.10.0 // indirect github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.17.11 // indirect @@ -174,7 +177,7 @@ require ( github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/speakeasy-api/openapi-overlay v0.9.0 // indirect - github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/afero v1.14.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect @@ -226,7 +229,7 @@ require ( k8s.io/component-base v0.32.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-aggregator v0.32.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/kubectl v0.32.1 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect @@ -234,7 +237,8 @@ require ( sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/kustomize/api v0.18.0 // indirect sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 170bb29..4a9e053 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/forcedotcom/go-soql v0.0.0-20220705175410-00f698360bee h1:UViGyUS6N3GdlALmKBczIi/mXrKkpQcZRyk0Hd5IqvU= +github.com/forcedotcom/go-soql v0.0.0-20220705175410-00f698360bee/go.mod h1:bON16NgZr710tAa9hHPeSNoNihIEXDEbVWy6rKP6rL8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -187,6 +189,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= +github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -211,8 +215,8 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= -github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= -github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -292,7 +296,11 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI= +github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/k-capehart/go-salesforce/v2 v2.5.2 h1:KcTvQofM+RGfJ+ov0BqXiWw9XS00onHCLnXfrVk47/o= +github.com/k-capehart/go-salesforce/v2 v2.5.2/go.mod h1:XdA3KHaEoAGra10uJdnGhMqRxaY1cqK7llJUb1B6fAU= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -383,6 +391,7 @@ github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmt github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -450,8 +459,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg= github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= @@ -574,6 +583,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191204025024-5ee1b9f4859a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -635,6 +645,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -691,6 +702,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= @@ -702,6 +714,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -715,8 +728,8 @@ k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= -k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= -k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY= +k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= k8s.io/cli-runtime v0.32.1 h1:19nwZPlYGJPUDbhAxDIS2/oydCikvKMHsxroKNGA2mM= @@ -729,8 +742,8 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-aggregator v0.32.1 h1:cztPyIHbo6tgrhYHDqmdmvxUufJKuxgAC/vog7yeWek= k8s.io/kube-aggregator v0.32.1/go.mod h1:sXjL5T8FO/rlBzTbBhahw9V5Nnr1UtzZHKTj9WxQCOU= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/kubectl v0.32.1 h1:/btLtXLQUU1rWx8AEvX9jrb9LaI6yeezt3sFALhB8M8= k8s.io/kubectl v0.32.1/go.mod h1:sezNuyWi1STk4ZNPVRIFfgjqMI6XMf+oCVLjZen/pFQ= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= @@ -745,7 +758,10 @@ sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= From 4ff5a89f4baf1e37cd12875d123bf81940d757c9 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Wed, 16 Jul 2025 00:17:17 -0700 Subject: [PATCH 2/8] fix ai feedback --- cmd/ctrlc/root/sync/salesforce/README.md | 12 +++++++++--- cmd/ctrlc/root/sync/salesforce/common/util.go | 6 ++++-- .../sync/salesforce/opportunities/opportunities.go | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cmd/ctrlc/root/sync/salesforce/README.md b/cmd/ctrlc/root/sync/salesforce/README.md index 733f63a..6a60df7 100644 --- a/cmd/ctrlc/root/sync/salesforce/README.md +++ b/cmd/ctrlc/root/sync/salesforce/README.md @@ -101,9 +101,12 @@ ctrlc sync salesforce accounts \ **Note**: - Metadata values are always stored as strings, so all field values are automatically converted -- The format is `ctrlplane/key=SalesforceField` for metadata mappings +- The format is `prefix/key=SalesforceField` for metadata mappings where: + - `ctrlplane/` prefix is for system fields (e.g., `ctrlplane/external-id`) + - `account/` prefix is for account-specific metadata + - Use custom prefixes as needed for your organization - The sync automatically includes common fields with default mappings that can be overridden -- The `MasterRecordId` is mapped to `ctrlplane/external-id` by default +- The `Id` is mapped to `ctrlplane/external-id` by default for accounts - All fields are fetched from Salesforce, so custom fields are always available for mapping ### Syncing Opportunities @@ -149,7 +152,10 @@ ctrlc sync salesforce opportunities \ **Note**: - Metadata values are always stored as strings, so all field values are automatically converted -- The format is `ctrlplane/key=SalesforceField` for metadata mappings +- The format is `prefix/key=SalesforceField` for metadata mappings where: + - `ctrlplane/` prefix is for system fields (e.g., `ctrlplane/external-id`) + - `opportunity/` prefix is for opportunity-specific metadata + - Use custom prefixes as needed for your organization - The sync automatically includes common fields with default mappings that can be overridden ## Resource Schema diff --git a/cmd/ctrlc/root/sync/salesforce/common/util.go b/cmd/ctrlc/root/sync/salesforce/common/util.go index 1cc2bc6..5b33cff 100644 --- a/cmd/ctrlc/root/sync/salesforce/common/util.go +++ b/cmd/ctrlc/root/sync/salesforce/common/util.go @@ -7,6 +7,7 @@ import ( "io" "net/url" "reflect" + "strconv" "strings" "github.com/charmbracelet/log" @@ -136,7 +137,7 @@ func fieldToString(fieldValue reflect.Value) string { } case reflect.Float64, reflect.Float32: if val := fieldValue.Float(); val != 0 { - return fmt.Sprintf("%g", val) // %g removes trailing zeros + return strconv.FormatFloat(val, 'f', -1, 64) } case reflect.Bool: if fieldValue.Bool() { @@ -261,12 +262,13 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec if err != nil { return fmt.Errorf("failed to query %s: %w", objectName, err) } - defer queryResp.Body.Close() body, err := io.ReadAll(queryResp.Body) if err != nil { + queryResp.Body.Close() return fmt.Errorf("failed to read response body: %w", err) } + queryResp.Body.Close() var queryResult struct { TotalSize int `json:"totalSize"` diff --git a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go index 0b427a4..0a97044 100644 --- a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go +++ b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go @@ -234,7 +234,7 @@ func transformOpportunityToResource(opportunity Opportunity, metadataMappings [] config := map[string]interface{}{ // Common cross-provider fields "name": opportunity.Name, - "amount": strconv.FormatFloat(opportunity.Amount, 'f', 2, 64), + "amount": strconv.FormatFloat(opportunity.Amount, 'f', -1, 64), "stage": opportunity.StageName, "id": opportunity.ID, @@ -243,7 +243,7 @@ func transformOpportunityToResource(opportunity Opportunity, metadataMappings [] "recordId": opportunity.ID, "closeDate": closeDateFormatted, "accountId": opportunity.AccountID, - "probability": strconv.FormatFloat(opportunity.Probability, 'f', 2, 64), + "probability": strconv.FormatFloat(opportunity.Probability, 'f', -1, 64), "type": opportunity.Type, "description": opportunity.Description, "nextStep": opportunity.NextStep, From 1f1761eb4c1310a26977e0e888afaf8b63d246e0 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Wed, 16 Jul 2025 11:15:37 -0700 Subject: [PATCH 3/8] viper updates and refactor of the shared logic --- cmd/ctrlc/root/sync/salesforce/README.md | 284 ++++++++++++------ .../root/sync/salesforce/accounts/accounts.go | 140 ++++----- .../root/sync/salesforce/common/client.go | 22 -- cmd/ctrlc/root/sync/salesforce/common/util.go | 220 ++++---------- .../salesforce/opportunities/opportunities.go | 177 ++++++----- cmd/ctrlc/root/sync/salesforce/salesforce.go | 27 +- 6 files changed, 425 insertions(+), 445 deletions(-) diff --git a/cmd/ctrlc/root/sync/salesforce/README.md b/cmd/ctrlc/root/sync/salesforce/README.md index 6a60df7..4c5409e 100644 --- a/cmd/ctrlc/root/sync/salesforce/README.md +++ b/cmd/ctrlc/root/sync/salesforce/README.md @@ -7,7 +7,7 @@ This package provides functionality to sync Salesforce CRM data into Ctrlplane a ### Prerequisites You need Salesforce OAuth2 credentials: -- **Domain**: Your Salesforce instance URL (e.g., `https://my-domain.my.salesforce.com`) +- **Domain**: Your Salesforce instance URL (e.g., `https://mycompany.my.salesforce.com`) - **Consumer Key**: From your Salesforce Connected App - **Consumer Secret**: From your Salesforce Connected App @@ -24,10 +24,12 @@ You need Salesforce OAuth2 credentials: ### Authentication +The Salesforce credentials are configured at the parent `salesforce` command level and apply to all subcommands (accounts, opportunities, etc.). + You can provide credentials via environment variables: ```bash -export SALESFORCE_DOMAIN="https://my-domain.my.salesforce.com" +export SALESFORCE_DOMAIN="https://mycompany.my.salesforce.com" export SALESFORCE_CONSUMER_KEY="your-consumer-key" export SALESFORCE_CONSUMER_SECRET="your-consumer-secret" ``` @@ -36,21 +38,28 @@ Or via command-line flags: ```bash ctrlc sync salesforce accounts \ - --domain "https://my-domain.my.salesforce.com" \ - --consumer-key "your-consumer-key" \ - --consumer-secret "your-consumer-secret" + --salesforce-domain "https://mycompany.my.salesforce.com" \ + --salesforce-consumer-key "your-consumer-key" \ + --salesforce-consumer-secret "your-consumer-secret" ``` ### Command-Line Flags +#### Global Salesforce Flags (apply to all subcommands) + +| Flag | Description | Default | +|------|-------------|---------| +| `--salesforce-domain` | Salesforce instance URL | `$SALESFORCE_DOMAIN` | +| `--salesforce-consumer-key` | OAuth2 consumer key | `$SALESFORCE_CONSUMER_KEY` | +| `--salesforce-consumer-secret` | OAuth2 consumer secret | `$SALESFORCE_CONSUMER_SECRET` | + +#### Subcommand Flags + Both `accounts` and `opportunities` commands support the following flags: | Flag | Description | Default | |------|-------------|---------| -| `--domain` | Salesforce instance URL | `$SALESFORCE_DOMAIN` | -| `--consumer-key` | OAuth2 consumer key | `$SALESFORCE_CONSUMER_KEY` | -| `--consumer-secret` | OAuth2 consumer secret | `$SALESFORCE_CONSUMER_SECRET` | -| `--provider`, `-p` | Resource provider name | `salesforce-accounts` or `salesforce-opportunities` | +| `--provider`, `-p` | Resource provider name | Auto-generated from domain (e.g., `wandb-salesforce-accounts`) | | `--metadata` | Custom metadata mappings (can be used multiple times) | Built-in defaults | | `--where` | SOQL WHERE clause to filter records | None (syncs all records) | | `--limit` | Maximum number of records to sync | 0 (no limit) | @@ -90,24 +99,27 @@ You can map any Salesforce field (including custom fields) to Ctrlplane metadata ```bash # Map standard and custom fields to metadata ctrlc sync salesforce accounts \ - --metadata="account/id=Id" \ - --metadata="ctrlplane/external-id=MasterRecordId" \ - --metadata="account/owner-id=OwnerId" \ --metadata="account/tier=Tier__c" \ --metadata="account/region=Region__c" \ - --metadata="account/annual-revenue=Annual_Revenue__c" \ - --metadata="account/health=Customer_Health__c" + --metadata="account/annual-revenue=AnnualRevenue" \ + --metadata="account/health=Customer_Health__c" \ + --metadata="account/contract-value=Contract_Value__c" ``` -**Note**: -- Metadata values are always stored as strings, so all field values are automatically converted -- The format is `prefix/key=SalesforceField` for metadata mappings where: - - `ctrlplane/` prefix is for system fields (e.g., `ctrlplane/external-id`) - - `account/` prefix is for account-specific metadata - - Use custom prefixes as needed for your organization -- The sync automatically includes common fields with default mappings that can be overridden -- The `Id` is mapped to `ctrlplane/external-id` by default for accounts -- All fields are fetched from Salesforce, so custom fields are always available for mapping +**Key Points about Metadata Mappings**: +- Format: `metadata-key=SalesforceFieldName` +- The left side is the metadata key in Ctrlplane (e.g., `account/tier`) +- The right side is the exact Salesforce field name (e.g., `Tier__c`) +- Custom fields in Salesforce typically end with `__c` +- All values are stored as strings in metadata +- Use `--list-all-fields` to discover available field names + +### Default Provider Naming + +If you don't specify a `--provider` name, the system automatically generates one based on your Salesforce domain: +- `https://wandb.my.salesforce.com` → `wandb-salesforce-accounts` +- `https://acme.my.salesforce.com` → `acme-salesforce-accounts` +- `https://mycompany.my.salesforce.com` → `mycompany-salesforce-accounts` ### Syncing Opportunities @@ -143,21 +155,13 @@ Just like accounts, you can map any Salesforce opportunity field (including cust ```bash # Map standard and custom fields to metadata ctrlc sync salesforce opportunities \ - --metadata="opportunity/id=Id" \ - --metadata="opportunity/account-id=AccountId" \ --metadata="opportunity/type=Type__c" \ --metadata="opportunity/expected-revenue=ExpectedRevenue" \ - --metadata="opportunity/lead-source=LeadSource" + --metadata="opportunity/lead-source=LeadSource" \ + --metadata="opportunity/next-step=NextStep" \ + --metadata="opportunity/use-case=Use_Case__c" ``` -**Note**: -- Metadata values are always stored as strings, so all field values are automatically converted -- The format is `prefix/key=SalesforceField` for metadata mappings where: - - `ctrlplane/` prefix is for system fields (e.g., `ctrlplane/external-id`) - - `opportunity/` prefix is for opportunity-specific metadata - - Use custom prefixes as needed for your organization -- The sync automatically includes common fields with default mappings that can be overridden - ## Resource Schema ### Salesforce Account Resource @@ -168,25 +172,65 @@ Resources are created with the following structure: { "version": "ctrlplane.dev/crm/account/v1", "kind": "SalesforceAccount", - "name": "Account Name", + "name": "Acme Corporation", "identifier": "001XX000003DHPh", "config": { - "name": "Account Name", + "name": "Acme Corporation", "industry": "Technology", "id": "001XX000003DHPh", + "type": "Customer", + "phone": "+1-555-0123", + "website": "https://acme.com", "salesforceAccount": { "recordId": "001XX000003DHPh", "ownerId": "005XX000001SvogAAC", - "billingCity": "San Francisco", - "website": "https://example.com" + "parentId": "", + "type": "Customer", + "accountSource": "Web", + "numberOfEmployees": 5000, + "description": "Major technology customer", + "billingAddress": { + "street": "123 Main St", + "city": "San Francisco", + "state": "CA", + "postalCode": "94105", + "country": "USA", + "latitude": 37.7749, + "longitude": -122.4194 + }, + "shippingAddress": { + "street": "123 Main St", + "city": "San Francisco", + "state": "CA", + "postalCode": "94105", + "country": "USA", + "latitude": 37.7749, + "longitude": -122.4194 + }, + "createdDate": "2023-01-15T10:30:00Z", + "lastModifiedDate": "2024-01-20T15:45:00Z", + "isDeleted": false, + "photoUrl": "https://..." } }, "metadata": { - "salesforce.account.id": "001XX000003DHPh", - "salesforce.account.owner_id": "005XX000001SvogAAC", - "salesforce.account.industry": "Technology", - "salesforce.account.billing_city": "San Francisco", - "salesforce.account.website": "https://example.com" + "ctrlplane/external-id": "001XX000003DHPh", + "account/id": "001XX000003DHPh", + "account/owner-id": "005XX000001SvogAAC", + "account/industry": "Technology", + "account/billing-city": "San Francisco", + "account/billing-state": "CA", + "account/billing-country": "USA", + "account/website": "https://acme.com", + "account/phone": "+1-555-0123", + "account/type": "Customer", + "account/source": "Web", + "account/shipping-city": "San Francisco", + "account/parent-id": "", + "account/employees": "5000", + // Custom fields added via --metadata mappings + "account/tier": "Enterprise", + "account/health": "Green" } } ``` @@ -197,27 +241,50 @@ Resources are created with the following structure: { "version": "ctrlplane.dev/crm/opportunity/v1", "kind": "SalesforceOpportunity", - "name": "Opportunity Name", + "name": "Acme Corp - Enterprise Deal", "identifier": "006XX000003DHPh", "config": { - "name": "Opportunity Name", - "amount": "50000.00", - "stage": "Qualification", + "name": "Acme Corp - Enterprise Deal", + "amount": 250000, + "stage": "Negotiation/Review", "id": "006XX000003DHPh", + "probability": 75, + "isClosed": false, + "isWon": false, "salesforceOpportunity": { "recordId": "006XX000003DHPh", - "closeDate": "2024-12-31T00:00:00Z", "accountId": "001XX000003DHPh", - "probability": "10" + "ownerId": "005XX000001SvogAAC", + "type": "New Business", + "leadSource": "Partner Referral", + "closeDate": "2024-12-31T00:00:00Z", + "forecastCategory": "Commit", + "description": "Enterprise license upgrade", + "nextStep": "Legal review", + "hasOpenActivity": true, + "createdDate": "2024-01-15T10:30:00Z", + "lastModifiedDate": "2024-02-20T15:45:00Z", + "lastActivityDate": "2024-02-19T00:00:00Z", + "fiscalQuarter": 4, + "fiscalYear": 2024 } }, "metadata": { - "salesforce.opportunity.id": "006XX000003DHPh", - "salesforce.opportunity.account_id": "001XX000003DHPh", - "salesforce.opportunity.stage": "Qualification", - "salesforce.opportunity.amount": "50000.00", - "salesforce.opportunity.probability": "10", - "salesforce.opportunity.close_date": "2024-12-31" + "ctrlplane/external-id": "006XX000003DHPh", + "opportunity/id": "006XX000003DHPh", + "opportunity/account-id": "001XX000003DHPh", + "opportunity/owner-id": "005XX000001SvogAAC", + "opportunity/stage": "Negotiation/Review", + "opportunity/amount": "250000", + "opportunity/probability": "75", + "opportunity/close-date": "2024-12-31", + "opportunity/type": "New Business", + "opportunity/lead-source": "Partner Referral", + "opportunity/is-closed": "false", + "opportunity/is-won": "false", + // Custom fields added via --metadata mappings + "opportunity/use-case": "Platform Migration", + "opportunity/competition": "Competitor X" } } ``` @@ -229,42 +296,85 @@ This integration uses the [go-salesforce](https://github.com/k-capehart/go-sales ### Features - **Dynamic Field Discovery**: Automatically discovers and fetches all available fields (standard and custom) from Salesforce objects -- **Custom Field Mappings**: Map any Salesforce field to Ctrlplane metadata using command-line flags +- **Custom Field Mappings**: Map any Salesforce field to Ctrlplane metadata using `--metadata` flags - **Flexible Filtering**: Use SOQL WHERE clauses with the `--where` flag to filter records -- **Flexible Transformations**: Use reflection-based utilities to handle field mappings dynamically -- **Extensible Architecture**: Shared utilities in `common/util.go` make it easy to add support for new Salesforce objects -- **Automatic Pagination**: Fetches all records by default, with support for limiting the number of records -- **Smart Field Capture**: Automatically captures any custom Salesforce fields not defined in the struct -- **Optional Field Listing**: Use `--list-all-fields` to see all available fields in the logs - -### Currently Syncs - -- **Accounts**: Complete account information including: - - Basic fields (name, industry, website, phone) - - Address information (billing and shipping) - - Hierarchy (parent/child relationships) - - Custom fields (any field ending in `__c`) - -- **Opportunities**: Deal information including: - - Basic fields (name, amount, stage, close date) - - Relationships (account associations) - - Probability and forecast data - - Custom fields (any field ending in `__c`) +- **Smart Field Capture**: Automatically captures any custom Salesforce fields (ending in `__c`) not defined in the struct +- **Automatic Pagination**: Handles large datasets efficiently with ID-based pagination +- **Subdomain-based Naming**: Automatically generates provider names from your Salesforce subdomain +- **Required Field Validation**: Uses Cobra's `MarkFlagRequired` for proper validation -### Pagination +### Core Architecture + +The sync implementation follows a clean, modular architecture: + +1. **Parse Metadata Mappings**: `ParseMetadataMappings` parses the `--metadata` flags once, returning: + - Field names to include in the SOQL query + - A lookup map for transforming fields to metadata keys + +2. **Query Salesforce**: `QuerySalesforceObject` performs the actual SOQL query with: + - Dynamic field selection based on struct tags and metadata mappings + - Automatic pagination handling + - Optional field listing for discovery -By default, the sync will retrieve all records from Salesforce using pagination: -- Records are fetched in batches of 2000 (Salesforce's default) -- Uses ID-based pagination to handle large datasets (avoids Salesforce's OFFSET limitation) -- Use the `--limit` flag to restrict the number of records synced -- Records are ordered by ID for consistent pagination +3. **Transform to Resources**: Each object is transformed into a Ctrlplane resource with: + - Standard metadata mappings + - Custom field mappings from the `--metadata` flags + - Proper type conversion (all metadata values are strings) + +4. **Upload to Ctrlplane**: `UpsertToCtrlplane` handles the resource upload + +### Handling Custom Fields + +Salesforce custom fields (typically ending in `__c`) are handled through: + +1. **Automatic Capture**: The `UnmarshalJSON` method captures any fields not in the struct into a `CustomFields` map +2. **Metadata Mapping**: Use `--metadata` flags to map these fields to Ctrlplane metadata +3. **Field Discovery**: Use `--list-all-fields` to see all available fields in your Salesforce instance + +Example workflow: +```bash +# 1. Discover available fields +ctrlc sync salesforce accounts --list-all-fields --limit 1 + +# 2. Map the custom fields you need +ctrlc sync salesforce accounts \ + --metadata="account/tier=Customer_Tier__c" \ + --metadata="account/segment=Market_Segment__c" \ + --metadata="account/arr=Annual_Recurring_Revenue__c" +``` ### Shared Utilities -The `common/util.go` file provides reusable functions for all Salesforce object syncs: -- `QuerySalesforceObject`: Generic function to query any Salesforce object with pagination support -- `UnmarshalWithCustomFields`: Captures any fields from Salesforce that aren't defined in the Go struct -- `GetKnownFieldsFromStruct`: Automatically extracts field names from struct tags -- `ParseMappings`: Handles custom metadata field mappings +The `common/` package provides reusable functions for all Salesforce object syncs: + +- **`InitSalesforceClient`**: Sets up OAuth2 authentication +- **`ParseMetadataMappings`**: Parses `--metadata` flags into field lists and lookup maps +- **`QuerySalesforceObject`**: Generic SOQL query with pagination +- **`GetCustomFieldValue`**: Gets any field value from struct (standard or custom) +- **`UnmarshalWithCustomFields`**: Captures unknown fields from Salesforce +- **`GetKnownFieldsFromStruct`**: Extracts field names from struct tags +- **`GetSalesforceSubdomain`**: Extracts subdomain for default provider naming +- **`UpsertToCtrlplane`**: Handles resource upload to Ctrlplane + +### Pagination + +Records are fetched efficiently using Salesforce best practices: +- ID-based pagination (avoids OFFSET limitations) +- Configurable batch size (default: 2000 records) +- Ordered by ID for consistent results +- Use `--limit` to restrict total records synced + +### Adding New Salesforce Objects + +To add support for a new Salesforce object (e.g., Leads): + +1. Create a new struct with JSON tags matching Salesforce field names +2. Include a `CustomFields map[string]interface{}` field +3. Implement `UnmarshalJSON` to capture custom fields +4. Create a command that: + - Uses `ParseMetadataMappings` for field mappings + - Calls `QuerySalesforceObject` for data retrieval + - Transforms objects to Ctrlplane resources + - Uses `UpsertToCtrlplane` for upload -These utilities make it easy to add support for new Salesforce objects (Leads, Contacts, etc.) with minimal code duplication. \ No newline at end of file +The shared utilities handle most of the complexity, making new object support straightforward. \ No newline at end of file diff --git a/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go index 6f20c44..1bbcf85 100644 --- a/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go +++ b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go @@ -2,7 +2,9 @@ package accounts import ( "context" + "fmt" "reflect" + "strconv" "github.com/MakeNowJust/heredoc/v2" "github.com/charmbracelet/log" @@ -10,6 +12,7 @@ import ( "github.com/ctrlplanedev/cli/internal/api" "github.com/k-capehart/go-salesforce/v2" "github.com/spf13/cobra" + "github.com/spf13/viper" ) type Account struct { @@ -64,11 +67,6 @@ type Account struct { CustomFields map[string]interface{} `json:"-"` } -// GetCustomFields implements the DynamicFieldHolder interface -func (a Account) GetCustomFields() map[string]interface{} { - return a.CustomFields -} - func (a *Account) UnmarshalJSON(data []byte) error { type Alias Account aux := &struct { @@ -91,9 +89,6 @@ func (a *Account) UnmarshalJSON(data []byte) error { func NewSalesforceAccountsCmd() *cobra.Command { var name string - var domain string - var consumerKey string - var consumerSecret string var metadataMappings []string var limit int var listAllFields bool @@ -103,10 +98,11 @@ func NewSalesforceAccountsCmd() *cobra.Command { Use: "accounts", Short: "Sync Salesforce accounts into Ctrlplane", Example: heredoc.Doc(` - # Make sure Salesforce credentials are configured via environment variables - # Sync all Salesforce accounts - $ ctrlc sync salesforce accounts + $ ctrlc sync salesforce accounts \ + --salesforce-domain="https://mycompany.my.salesforce.com" \ + --salesforce-consumer-key="your-key" \ + --salesforce-consumer-secret="your-secret" # Sync accounts with a specific filter $ ctrlc sync salesforce accounts --where="Customer_Health__c != null" @@ -131,18 +127,41 @@ func NewSalesforceAccountsCmd() *cobra.Command { # Combine filters with metadata mappings $ ctrlc sync salesforce accounts \ + --salesforce-domain="https://mycompany.my.salesforce.com" \ + --salesforce-consumer-key="your-key" \ + --salesforce-consumer-secret="your-secret" \ --where="Type = 'Customer' AND AnnualRevenue > 1000000" \ --metadata="account/revenue=AnnualRevenue" `), - PreRunE: common.ValidateFlags(&domain, &consumerKey, &consumerSecret), - RunE: runSync(&name, &domain, &consumerKey, &consumerSecret, &metadataMappings, &limit, &listAllFields, &whereClause), + RunE: func(cmd *cobra.Command, args []string) error { + domain := viper.GetString("salesforce-domain") + consumerKey := viper.GetString("salesforce-consumer-key") + consumerSecret := viper.GetString("salesforce-consumer-secret") + + log.Info("Syncing Salesforce accounts into Ctrlplane", "domain", domain) + + ctx := context.Background() + + sf, err := common.InitSalesforceClient(domain, consumerKey, consumerSecret) + if err != nil { + return err + } + + resources, err := processAccounts(ctx, sf, metadataMappings, limit, listAllFields, whereClause) + if err != nil { + return err + } + + if name == "" { + subdomain := common.GetSalesforceSubdomain(domain) + name = fmt.Sprintf("%s-salesforce-accounts", subdomain) + } + + return common.UpsertToCtrlplane(ctx, resources, name) + }, } - // Add command flags cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") - cmd.Flags().StringVar(&domain, "domain", "", "Salesforce domain (e.g., https://my-domain.my.salesforce.com)") - cmd.Flags().StringVar(&consumerKey, "consumer-key", "", "Salesforce consumer key") - cmd.Flags().StringVar(&consumerSecret, "consumer-secret", "", "Salesforce consumer secret") cmd.Flags().StringArrayVar(&metadataMappings, "metadata", []string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") cmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of records to sync (0 = no limit)") cmd.Flags().BoolVar(&listAllFields, "list-all-fields", false, "List all available Salesforce fields in the logs") @@ -151,35 +170,11 @@ func NewSalesforceAccountsCmd() *cobra.Command { return cmd } -// runSync contains the main sync logic -func runSync(name, domain, consumerKey, consumerSecret *string, metadataMappings *[]string, limit *int, listAllFields *bool, whereClause *string) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - log.Info("Syncing Salesforce accounts into Ctrlplane", "domain", *domain) - - ctx := context.Background() - - sf, err := common.InitSalesforceClient(*domain, *consumerKey, *consumerSecret) - if err != nil { - return err - } - - resources, err := processAccounts(ctx, sf, *metadataMappings, *limit, *listAllFields, *whereClause) - if err != nil { - return err - } - - if *name == "" { - *name = "salesforce-accounts" - } - - return common.UpsertToCtrlplane(ctx, resources, *name) - } -} - -// processAccounts queries and transforms accounts func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMappings []string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { + additionalFields, mappingLookup := common.ParseMetadataMappings(metadataMappings) - accounts, err := queryAccounts(ctx, sf, limit, listAllFields, metadataMappings, whereClause) + var accounts []Account + err := common.QuerySalesforceObject(ctx, sf, "Account", limit, listAllFields, &accounts, additionalFields, whereClause) if err != nil { return nil, err } @@ -188,52 +183,38 @@ func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMap resources := []api.CreateResource{} for _, account := range accounts { - resource := transformAccountToResource(account, metadataMappings) + resource := transformAccountToResource(account, mappingLookup) resources = append(resources, resource) } return resources, nil } -func queryAccounts(ctx context.Context, sf *salesforce.Salesforce, limit int, listAllFields bool, metadataMappings []string, whereClause string) ([]Account, error) { - additionalFields := common.ExtractFieldsFromMetadataMappings(metadataMappings) - - var accounts []Account - err := common.QuerySalesforceObject(ctx, sf, "Account", limit, listAllFields, &accounts, additionalFields, whereClause) - if err != nil { - return nil, err +func transformAccountToResource(account Account, mappingLookup map[string]string) api.CreateResource { + metadata := map[string]string{ + "ctrlplane/external-id": account.ID, + "account/id": account.ID, + "account/owner-id": account.OwnerId, + "account/industry": account.Industry, + "account/billing-city": account.BillingCity, + "account/billing-state": account.BillingState, + "account/billing-country": account.BillingCountry, + "account/website": account.Website, + "account/phone": account.Phone, + "account/type": account.Type, + "account/source": account.AccountSource, + "account/shipping-city": account.ShippingCity, + "account/parent-id": account.ParentId, + "account/employees": strconv.Itoa(account.NumberOfEmployees), } - return accounts, nil -} -func transformAccountToResource(account Account, metadataMappings []string) api.CreateResource { - defaultMetadataMappings := map[string]string{ - "ctrlplane/external-id": "Id", - "account/id": "Id", - "account/owner-id": "OwnerId", - "account/industry": "Industry", - "account/billing-city": "BillingCity", - "account/billing-state": "BillingState", - "account/billing-country": "BillingCountry", - "account/website": "Website", - "account/phone": "Phone", - "account/type": "Type", - "account/source": "AccountSource", - "account/shipping-city": "ShippingCity", - "account/parent-id": "ParentId", - "account/employees": "NumberOfEmployees", - "account/region": "Region__c", - "account/annual-revenue": "Annual_Revenue__c", - "account/tier": "Tier__c", - "account/health": "Customer_Health__c", + for fieldName, metadataKey := range mappingLookup { + if value, found := common.GetCustomFieldValue(account, fieldName); found { + metadata[metadataKey] = value + } } - // Parse metadata mappings using common utility - metadata := common.ParseMappings(account, metadataMappings, defaultMetadataMappings) - - // Build base config with common fields config := map[string]interface{}{ - // Common cross-provider fields "name": account.Name, "industry": account.Industry, "id": account.ID, @@ -241,7 +222,6 @@ func transformAccountToResource(account Account, metadataMappings []string) api. "phone": account.Phone, "website": account.Website, - // Salesforce-specific implementation details "salesforceAccount": map[string]interface{}{ "recordId": account.ID, "ownerId": account.OwnerId, diff --git a/cmd/ctrlc/root/sync/salesforce/common/client.go b/cmd/ctrlc/root/sync/salesforce/common/client.go index 804830d..85128ca 100644 --- a/cmd/ctrlc/root/sync/salesforce/common/client.go +++ b/cmd/ctrlc/root/sync/salesforce/common/client.go @@ -2,10 +2,8 @@ package common import ( "fmt" - "os" "github.com/k-capehart/go-salesforce/v2" - "github.com/spf13/cobra" ) func InitSalesforceClient(domain, consumerKey, consumerSecret string) (*salesforce.Salesforce, error) { @@ -19,23 +17,3 @@ func InitSalesforceClient(domain, consumerKey, consumerSecret string) (*salesfor } return sf, nil } - -func ValidateFlags(domain, consumerKey, consumerSecret *string) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - if *domain == "" { - *domain = os.Getenv("SALESFORCE_DOMAIN") - } - if *consumerKey == "" { - *consumerKey = os.Getenv("SALESFORCE_CONSUMER_KEY") - } - if *consumerSecret == "" { - *consumerSecret = os.Getenv("SALESFORCE_CONSUMER_SECRET") - } - - if *domain == "" || *consumerKey == "" || *consumerSecret == "" { - return fmt.Errorf("Salesforce credentials are required. Set SALESFORCE_DOMAIN, SALESFORCE_CONSUMER_KEY, and SALESFORCE_CONSUMER_SECRET environment variables or use flags") - } - - return nil - } -} diff --git a/cmd/ctrlc/root/sync/salesforce/common/util.go b/cmd/ctrlc/root/sync/salesforce/common/util.go index 5b33cff..86264d2 100644 --- a/cmd/ctrlc/root/sync/salesforce/common/util.go +++ b/cmd/ctrlc/root/sync/salesforce/common/util.go @@ -7,7 +7,6 @@ import ( "io" "net/url" "reflect" - "strconv" "strings" "github.com/charmbracelet/log" @@ -16,22 +15,31 @@ import ( "github.com/spf13/viper" ) -// DynamicFieldHolder interface for structs that can hold custom fields -type DynamicFieldHolder interface { - GetCustomFields() map[string]interface{} +func GetSalesforceSubdomain(domain string) string { + subdomain := "salesforce" + if strings.HasPrefix(domain, "https://") || strings.HasPrefix(domain, "http://") { + parts := strings.Split(domain, "//") + if len(parts) > 1 { + hostParts := strings.Split(parts[1], ".") + if len(hostParts) > 0 { + subdomain = hostParts[0] + } + } + } + return subdomain } -// ExtractFieldsFromMetadataMappings extracts Salesforce field names from metadata mappings -func ExtractFieldsFromMetadataMappings(metadataMappings []string) []string { +func ParseMetadataMappings(mappings []string) ([]string, map[string]string) { fieldMap := make(map[string]bool) + lookupMap := make(map[string]string) // fieldName -> metadataKey - for _, mapping := range metadataMappings { - parts := strings.SplitN(mapping, "=", 2) + for _, mapping := range mappings { + parts := strings.Split(mapping, "=") if len(parts) == 2 { - salesforceField := strings.TrimSpace(parts[1]) - if salesforceField != "" { - fieldMap[salesforceField] = true - } + metadataKey := parts[0] + fieldName := parts[1] + fieldMap[fieldName] = true + lookupMap[fieldName] = metadataKey } } @@ -40,120 +48,37 @@ func ExtractFieldsFromMetadataMappings(metadataMappings []string) []string { fields = append(fields, field) } - return fields + return fields, lookupMap } -// ParseMappings applies custom field mappings to extract string values for metadata -func ParseMappings(data interface{}, mappings []string, defaultMappings map[string]string) map[string]string { - result := map[string]string{} - - dataValue := reflect.ValueOf(data) - dataType := reflect.TypeOf(data) - - if dataValue.Kind() == reflect.Ptr { - dataValue = dataValue.Elem() - dataType = dataType.Elem() +func GetCustomFieldValue(obj interface{}, fieldName string) (string, bool) { + objValue := reflect.ValueOf(obj) + if objValue.Kind() == reflect.Ptr { + objValue = objValue.Elem() } - // Process custom mappings (format: ctrlplane/key=SalesforceField) - for _, mapping := range mappings { - parts := strings.SplitN(mapping, "=", 2) - if len(parts) != 2 { - log.Warn("Invalid mapping format, skipping", "mapping", mapping) - continue - } - - ctrlplaneKey, sfField := parts[0], parts[1] - - found := false - for i := 0; i < dataType.NumField(); i++ { - field := dataType.Field(i) - jsonTag := field.Tag.Get("json") - - if jsonTag == sfField { - fieldValue := dataValue.Field(i) - - strValue := fieldToString(fieldValue) - - if strValue != "" { - result[ctrlplaneKey] = strValue - } - found = true - break - } - } - - // If not found in struct fields, check CustomFields if the struct implements DynamicFieldHolder - if !found { - if holder, ok := data.(DynamicFieldHolder); ok { - customFields := holder.GetCustomFields() - if customFields != nil { - if value, exists := customFields[sfField]; exists { - strValue := fmt.Sprintf("%v", value) - if strValue != "" && strValue != "" { - result[ctrlplaneKey] = strValue - } - } - } - } + customFields := objValue.FieldByName("CustomFields") + if customFields.IsValid() && customFields.Kind() == reflect.Map { + if value := customFields.MapIndex(reflect.ValueOf(fieldName)); value.IsValid() { + return fmt.Sprintf("%v", value.Interface()), true } } - // Add default mappings if they haven't been explicitly mapped - existingKeys := make(map[string]bool) - for key := range result { - existingKeys[key] = true - } - - for defaultKey, sfField := range defaultMappings { - if !existingKeys[defaultKey] { - // Find and add the default field - for i := 0; i < dataType.NumField(); i++ { - field := dataType.Field(i) - if field.Tag.Get("json") == sfField { - fieldValue := dataValue.Field(i) - strValue := fieldToString(fieldValue) - - if strValue != "" { - result[defaultKey] = strValue - } - break + objType := objValue.Type() + for i := 0; i < objType.NumField(); i++ { + field := objType.Field(i) + if jsonTag := field.Tag.Get("json"); jsonTag != "" { + tagName := strings.Split(jsonTag, ",")[0] + if tagName == fieldName { + fieldValue := objValue.Field(i) + if fieldValue.IsValid() && fieldValue.CanInterface() { + return fmt.Sprintf("%v", fieldValue.Interface()), true } } } } - return result -} - -// fieldToString converts a reflect.Value to string for metadata -func fieldToString(fieldValue reflect.Value) string { - switch fieldValue.Kind() { - case reflect.String: - return fieldValue.String() - case reflect.Int, reflect.Int64: - if val := fieldValue.Int(); val != 0 { - return fmt.Sprintf("%d", val) - } - case reflect.Float64, reflect.Float32: - if val := fieldValue.Float(); val != 0 { - return strconv.FormatFloat(val, 'f', -1, 64) - } - case reflect.Bool: - if fieldValue.Bool() { - return "true" - } - return "false" - default: - // For complex types, try to marshal to JSON - if fieldValue.IsValid() && fieldValue.CanInterface() { - if bytes, err := json.Marshal(fieldValue.Interface()); err == nil { - return string(bytes) - } - } - } - - return "" + return "", false } // QuerySalesforceObject performs a generic query on any Salesforce object with pagination support @@ -164,7 +89,6 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec } elementType := targetValue.Type().Elem() - // Get field names from the struct fieldNames := []string{} for i := 0; i < elementType.NumField(); i++ { field := elementType.Field(i) @@ -177,7 +101,6 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec } } - // Include additional fields passed in (e.g., from metadata mappings) for _, field := range additionalFields { found := false for _, existing := range fieldNames { @@ -191,7 +114,6 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec } } - // If listAllFields is true, describe the object to show what's available if listAllFields { describeResp, err := sf.DoRequest("GET", fmt.Sprintf("/sobjects/%s/describe", objectName), nil) if err != nil { @@ -204,7 +126,6 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec return fmt.Errorf("failed to decode describe response: %w", err) } - // Extract all available field names for logging fields, ok := describeResult["fields"].([]interface{}) if !ok { return fmt.Errorf("unexpected describe response format") @@ -223,12 +144,10 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec log.Info("Available fields", "object", objectName, "count", len(allFieldNames), "fields", allFieldNames) } - // Build query with pagination support totalRetrieved := 0 lastId := "" batchSize := 2000 - // Query in batches using ID-based pagination to avoid OFFSET limits for { fieldsClause := strings.Join(fieldNames, ", ") baseQuery := fmt.Sprintf("SELECT %s FROM %s", fieldsClause, objectName) @@ -283,7 +202,6 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec batchSlice := reflect.New(targetValue.Type()).Elem() - // Unmarshal records into the batch slice - this will trigger our custom UnmarshalJSON if err := json.Unmarshal(queryResult.Records, batchSlice.Addr().Interface()); err != nil { return fmt.Errorf("failed to unmarshal records: %w", err) } @@ -330,24 +248,6 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec return nil } -// ExtractCustomFields extracts fields from a JSON response that aren't in the struct -func ExtractCustomFields(data []byte, knownFields map[string]bool) (map[string]interface{}, error) { - var allFields map[string]interface{} - if err := json.Unmarshal(data, &allFields); err != nil { - return nil, err - } - - customFields := make(map[string]interface{}) - for key, value := range allFields { - if !knownFields[key] { - customFields[key] = value - } - } - - return customFields, nil -} - -// GetKnownFieldsFromStruct extracts all JSON field names from a struct's tags func GetKnownFieldsFromStruct(structType reflect.Type) map[string]bool { knownFields := make(map[string]bool) @@ -372,7 +272,26 @@ func GetKnownFieldsFromStruct(structType reflect.Type) map[string]bool { return knownFields } -// UpsertToCtrlplane creates or updates a resource provider and sets its resources +func UnmarshalWithCustomFields(data []byte, target interface{}, knownFields map[string]bool) (map[string]interface{}, error) { + if err := json.Unmarshal(data, target); err != nil { + return nil, err + } + + var allFields map[string]interface{} + if err := json.Unmarshal(data, &allFields); err != nil { + return nil, err + } + + customFields := make(map[string]interface{}) + for fieldName, value := range allFields { + if !knownFields[fieldName] { + customFields[fieldName] = value + } + } + + return customFields, nil +} + func UpsertToCtrlplane(ctx context.Context, resources []api.CreateResource, providerName string) error { apiURL := viper.GetString("url") apiKey := viper.GetString("api-key") @@ -409,24 +328,3 @@ func UpsertToCtrlplane(ctx context.Context, resources []api.CreateResource, prov log.Info("Successfully synced resources", "count", len(resources)) return nil } - -// UnmarshalWithCustomFields unmarshals JSON data into the target struct and returns any unknown fields -func UnmarshalWithCustomFields(data []byte, target interface{}, knownFields map[string]bool) (map[string]interface{}, error) { - if err := json.Unmarshal(data, target); err != nil { - return nil, err - } - - var allFields map[string]interface{} - if err := json.Unmarshal(data, &allFields); err != nil { - return nil, err - } - - customFields := make(map[string]interface{}) - for fieldName, value := range allFields { - if !knownFields[fieldName] { - customFields[fieldName] = value - } - } - - return customFields, nil -} diff --git a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go index 0a97044..cee9659 100644 --- a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go +++ b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go @@ -2,6 +2,7 @@ package opportunities import ( "context" + "fmt" "reflect" "strconv" "time" @@ -12,6 +13,7 @@ import ( "github.com/ctrlplanedev/cli/internal/api" "github.com/k-capehart/go-salesforce/v2" "github.com/spf13/cobra" + "github.com/spf13/viper" ) type Opportunity struct { @@ -59,11 +61,6 @@ type Opportunity struct { CustomFields map[string]interface{} `json:"-"` } -// GetCustomFields implements the DynamicFieldHolder interface -func (o Opportunity) GetCustomFields() map[string]interface{} { - return o.CustomFields -} - // UnmarshalJSON implements custom unmarshalling to capture unknown fields func (o *Opportunity) UnmarshalJSON(data []byte) error { type Alias Opportunity @@ -87,9 +84,6 @@ func (o *Opportunity) UnmarshalJSON(data []byte) error { func NewSalesforceOpportunitiesCmd() *cobra.Command { var name string - var domain string - var consumerKey string - var consumerSecret string var metadataMappings []string var limit int var listAllFields bool @@ -100,127 +94,101 @@ func NewSalesforceOpportunitiesCmd() *cobra.Command { Short: "Sync Salesforce opportunities into Ctrlplane", Example: heredoc.Doc(` # Sync all Salesforce opportunities - $ ctrlc sync salesforce opportunities + $ ctrlc sync salesforce opportunities \ + --salesforce-domain="https://mycompany.my.salesforce.com" \ + --salesforce-consumer-key="your-key" \ + --salesforce-consumer-secret="your-secret" # Sync opportunities with a specific filter - $ ctrlc sync salesforce opportunities --where="IsWon = true" + $ ctrlc sync salesforce opportunities --where="Amount > 100000" + + # Sync opportunities and list all available fields in logs + $ ctrlc sync salesforce opportunities --list-all-fields # Sync opportunities with custom provider name $ ctrlc sync salesforce opportunities --provider my-salesforce-opportunities # Sync with custom metadata mappings $ ctrlc sync salesforce opportunities \ - --metadata="opportunity/stage=StageName" \ - --metadata="opportunity/amount=Amount" \ - --metadata="opportunity/close-date=CloseDate" + --metadata="opportunity/id=Id" \ + --metadata="opportunity/owner-id=OwnerId" \ + --metadata="opportunity/forecast=ForecastCategory" \ + --metadata="opportunity/stage-custom=Custom_Stage__c" # Sync with a limit on number of records - $ ctrlc sync salesforce opportunities --limit 100 + $ ctrlc sync salesforce opportunities --limit 500 # Combine filters with metadata mappings $ ctrlc sync salesforce opportunities \ - --where="Amount > 50000 AND StageName != 'Closed Lost'" \ - --metadata="opportunity/probability=Probability" + --salesforce-domain="https://mycompany.my.salesforce.com" \ + --salesforce-consumer-key="your-key" \ + --salesforce-consumer-secret="your-secret" \ + --where="StageName = 'Closed Won' AND Amount > 50000" \ + --metadata="opportunity/revenue=Amount" `), - PreRunE: common.ValidateFlags(&domain, &consumerKey, &consumerSecret), - RunE: runSync(&name, &domain, &consumerKey, &consumerSecret, &metadataMappings, &limit, &listAllFields, &whereClause), - } + RunE: func(cmd *cobra.Command, args []string) error { + domain := viper.GetString("salesforce-domain") + consumerKey := viper.GetString("salesforce-consumer-key") + consumerSecret := viper.GetString("salesforce-consumer-secret") - // Add command flags - cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") - cmd.Flags().StringVar(&domain, "domain", "", "Salesforce domain (e.g., https://my-domain.my.salesforce.com)") - cmd.Flags().StringVar(&consumerKey, "consumer-key", "", "Salesforce consumer key") - cmd.Flags().StringVar(&consumerSecret, "consumer-secret", "", "Salesforce consumer secret") - cmd.Flags().StringArrayVar(&metadataMappings, "metadata", []string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") - cmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of records to sync (0 = no limit)") - cmd.Flags().BoolVar(&listAllFields, "list-all-fields", false, "List all available Salesforce fields in the logs") - cmd.Flags().StringVar(&whereClause, "where", "", "SOQL WHERE clause to filter records (e.g., \"IsClosed = false\")") + log.Info("Syncing Salesforce opportunities into Ctrlplane", "domain", domain) - return cmd -} + ctx := context.Background() -// runSync contains the main sync logic -func runSync(name, domain, consumerKey, consumerSecret *string, metadataMappings *[]string, limit *int, listAllFields *bool, whereClause *string) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - log.Info("Syncing Salesforce opportunities into Ctrlplane", "domain", *domain) + sf, err := common.InitSalesforceClient(domain, consumerKey, consumerSecret) + if err != nil { + return err + } - ctx := context.Background() + resources, err := processOpportunities(ctx, sf, metadataMappings, limit, listAllFields, whereClause) + if err != nil { + return err + } - sf, err := common.InitSalesforceClient(*domain, *consumerKey, *consumerSecret) - if err != nil { - return err - } + if name == "" { + subdomain := common.GetSalesforceSubdomain(domain) + name = fmt.Sprintf("%s-salesforce-opportunities", subdomain) + } - resources, err := processOpportunities(ctx, sf, *metadataMappings, *limit, *listAllFields, *whereClause) - if err != nil { - return err - } + return common.UpsertToCtrlplane(ctx, resources, name) + }, + } - if *name == "" { - *name = "salesforce-opportunities" - } + cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") + cmd.Flags().StringArrayVar(&metadataMappings, "metadata", []string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") + cmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of records to sync (0 = no limit)") + cmd.Flags().BoolVar(&listAllFields, "list-all-fields", false, "List all available Salesforce fields in the logs") + cmd.Flags().StringVar(&whereClause, "where", "", "SOQL WHERE clause to filter records (e.g., \"Amount > 100000\")") - return common.UpsertToCtrlplane(ctx, resources, *name) - } + return cmd } // processOpportunities queries and transforms opportunities func processOpportunities(ctx context.Context, sf *salesforce.Salesforce, metadataMappings []string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { - opportunities, err := queryOpportunities(ctx, sf, limit, listAllFields, metadataMappings, whereClause) + // Parse metadata mappings to get field names for query + additionalFields, mappingLookup := common.ParseMetadataMappings(metadataMappings) + + // Query Salesforce for opportunities + var opportunities []Opportunity + err := common.QuerySalesforceObject(ctx, sf, "Opportunity", limit, listAllFields, &opportunities, additionalFields, whereClause) if err != nil { return nil, err } log.Info("Found Salesforce opportunities", "count", len(opportunities)) - // Transform to ctrlplane resources + // Transform opportunities to Ctrlplane resources resources := []api.CreateResource{} for _, opp := range opportunities { - resource := transformOpportunityToResource(opp, metadataMappings) + resource := transformOpportunityToResource(opp, mappingLookup) resources = append(resources, resource) } return resources, nil } -func queryOpportunities(ctx context.Context, sf *salesforce.Salesforce, limit int, listAllFields bool, metadataMappings []string, whereClause string) ([]Opportunity, error) { - // Extract Salesforce field names from metadata mappings - additionalFields := common.ExtractFieldsFromMetadataMappings(metadataMappings) - - var opportunities []Opportunity - err := common.QuerySalesforceObject(ctx, sf, "Opportunity", limit, listAllFields, &opportunities, additionalFields, whereClause) - if err != nil { - return nil, err - } - return opportunities, nil -} - -func transformOpportunityToResource(opportunity Opportunity, metadataMappings []string) api.CreateResource { - // Define default metadata mappings for opportunities - defaultMetadataMappings := map[string]string{ - "opportunity/id": "Id", - "ctrlplane/external-id": "Id", - "opportunity/account-id": "AccountId", - "opportunity/stage": "StageName", - "opportunity/amount": "Amount", - "opportunity/probability": "Probability", - "opportunity/close-date": "CloseDate", - "opportunity/name": "Name", - "opportunity/type": "Type", - "opportunity/owner-id": "OwnerId", - "opportunity/is-closed": "IsClosed", - "opportunity/is-won": "IsWon", - "opportunity/lead-source": "LeadSource", - "opportunity/forecast-category": "ForecastCategory", - "opportunity/contact-id": "ContactId", - "opportunity/campaign-id": "CampaignId", - "opportunity/created-date": "CreatedDate", - "opportunity/last-modified": "LastModifiedDate", - } - - // Parse metadata mappings using common utility - metadata := common.ParseMappings(opportunity, metadataMappings, defaultMetadataMappings) - +func transformOpportunityToResource(opportunity Opportunity, mappingLookup map[string]string) api.CreateResource { + // Format close date var closeDateFormatted string if opportunity.CloseDate != "" { if t, err := time.Parse("2006-01-02", opportunity.CloseDate); err == nil { @@ -230,15 +198,40 @@ func transformOpportunityToResource(opportunity Opportunity, metadataMappings [] } } - // Build base config with common fields + metadata := map[string]string{ + "opportunity/id": opportunity.ID, + "ctrlplane/external-id": opportunity.ID, + "opportunity/account-id": opportunity.AccountID, + "opportunity/stage": opportunity.StageName, + "opportunity/amount": strconv.FormatFloat(opportunity.Amount, 'f', -1, 64), + "opportunity/probability": strconv.FormatFloat(opportunity.Probability, 'f', -1, 64), + "opportunity/close-date": closeDateFormatted, + "opportunity/name": opportunity.Name, + "opportunity/type": opportunity.Type, + "opportunity/owner-id": opportunity.OwnerID, + "opportunity/is-closed": strconv.FormatBool(opportunity.IsClosed), + "opportunity/is-won": strconv.FormatBool(opportunity.IsWon), + "opportunity/lead-source": opportunity.LeadSource, + "opportunity/forecast-category": opportunity.ForecastCategory, + "opportunity/contact-id": opportunity.ContactId, + "opportunity/campaign-id": opportunity.CampaignID, + "opportunity/created-date": opportunity.CreatedDate, + "opportunity/last-modified": opportunity.LastModifiedDate, + } + + // Apply custom metadata mappings + for fieldName, metadataKey := range mappingLookup { + if value, found := common.GetCustomFieldValue(opportunity, fieldName); found { + metadata[metadataKey] = value + } + } + config := map[string]interface{}{ - // Common cross-provider fields "name": opportunity.Name, "amount": strconv.FormatFloat(opportunity.Amount, 'f', -1, 64), "stage": opportunity.StageName, "id": opportunity.ID, - // Salesforce-specific implementation details "salesforceOpportunity": map[string]interface{}{ "recordId": opportunity.ID, "closeDate": closeDateFormatted, diff --git a/cmd/ctrlc/root/sync/salesforce/salesforce.go b/cmd/ctrlc/root/sync/salesforce/salesforce.go index ecba6e3..7d3c635 100644 --- a/cmd/ctrlc/root/sync/salesforce/salesforce.go +++ b/cmd/ctrlc/root/sync/salesforce/salesforce.go @@ -5,6 +5,7 @@ import ( "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/accounts" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/opportunities" "github.com/spf13/cobra" + "github.com/spf13/viper" ) func NewSalesforceCmd() *cobra.Command { @@ -12,12 +13,32 @@ func NewSalesforceCmd() *cobra.Command { Use: "salesforce", Short: "Sync Salesforce resources into Ctrlplane", Example: heredoc.Doc(` - # Sync all Salesforce objects - $ ctrlc sync salesforce accounts - $ ctrlc sync salesforce opportunities + # Sync Salesforce accounts + $ ctrlc sync salesforce accounts \ + --salesforce-domain="https://mycompany.my.salesforce.com" \ + --salesforce-consumer-key="your-key" \ + --salesforce-consumer-secret="your-secret" + + # Sync Salesforce opportunities + $ ctrlc sync salesforce opportunities \ + --salesforce-domain="https://mycompany.my.salesforce.com" \ + --salesforce-consumer-key="your-key" \ + --salesforce-consumer-secret="your-secret" `), } + cmd.PersistentFlags().String("salesforce-domain", "", "Salesforce domain (e.g., https://my-domain.my.salesforce.com)") + cmd.PersistentFlags().String("salesforce-consumer-key", "", "Salesforce consumer key") + cmd.PersistentFlags().String("salesforce-consumer-secret", "", "Salesforce consumer secret") + + viper.BindPFlag("salesforce-domain", cmd.PersistentFlags().Lookup("salesforce-domain")) + viper.BindPFlag("salesforce-consumer-key", cmd.PersistentFlags().Lookup("salesforce-consumer-key")) + viper.BindPFlag("salesforce-consumer-secret", cmd.PersistentFlags().Lookup("salesforce-consumer-secret")) + + cmd.MarkPersistentFlagRequired("salesforce-domain") + cmd.MarkPersistentFlagRequired("salesforce-consumer-key") + cmd.MarkPersistentFlagRequired("salesforce-consumer-secret") + cmd.AddCommand(accounts.NewSalesforceAccountsCmd()) cmd.AddCommand(opportunities.NewSalesforceOpportunitiesCmd()) From f9644ffa1c783ce53251bea3e5885821ddbb5fe6 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Wed, 16 Jul 2025 11:35:31 -0700 Subject: [PATCH 4/8] refactor --- .../root/sync/salesforce/accounts/accounts.go | 17 +- cmd/ctrlc/root/sync/salesforce/common/util.go | 258 +++++++++--------- .../salesforce/opportunities/opportunities.go | 23 +- 3 files changed, 149 insertions(+), 149 deletions(-) diff --git a/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go index 1bbcf85..897c1c8 100644 --- a/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go +++ b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go @@ -89,7 +89,7 @@ func (a *Account) UnmarshalJSON(data []byte) error { func NewSalesforceAccountsCmd() *cobra.Command { var name string - var metadataMappings []string + var metadataMappings map[string]string var limit int var listAllFields bool var whereClause string @@ -162,7 +162,7 @@ func NewSalesforceAccountsCmd() *cobra.Command { } cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") - cmd.Flags().StringArrayVar(&metadataMappings, "metadata", []string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") + cmd.Flags().StringToStringVar(&metadataMappings, "metadata", map[string]string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") cmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of records to sync (0 = no limit)") cmd.Flags().BoolVar(&listAllFields, "list-all-fields", false, "List all available Salesforce fields in the logs") cmd.Flags().StringVar(&whereClause, "where", "", "SOQL WHERE clause to filter records (e.g., \"Customer_Health__c != null\")") @@ -170,8 +170,11 @@ func NewSalesforceAccountsCmd() *cobra.Command { return cmd } -func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMappings []string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { - additionalFields, mappingLookup := common.ParseMetadataMappings(metadataMappings) +func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMappings map[string]string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { + additionalFields := make([]string, 0, len(metadataMappings)) + for _, fieldName := range metadataMappings { + additionalFields = append(additionalFields, fieldName) + } var accounts []Account err := common.QuerySalesforceObject(ctx, sf, "Account", limit, listAllFields, &accounts, additionalFields, whereClause) @@ -183,14 +186,14 @@ func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMap resources := []api.CreateResource{} for _, account := range accounts { - resource := transformAccountToResource(account, mappingLookup) + resource := transformAccountToResource(account, metadataMappings) resources = append(resources, resource) } return resources, nil } -func transformAccountToResource(account Account, mappingLookup map[string]string) api.CreateResource { +func transformAccountToResource(account Account, metadataMappings map[string]string) api.CreateResource { metadata := map[string]string{ "ctrlplane/external-id": account.ID, "account/id": account.ID, @@ -208,7 +211,7 @@ func transformAccountToResource(account Account, mappingLookup map[string]string "account/employees": strconv.Itoa(account.NumberOfEmployees), } - for fieldName, metadataKey := range mappingLookup { + for metadataKey, fieldName := range metadataMappings { if value, found := common.GetCustomFieldValue(account, fieldName); found { metadata[metadataKey] = value } diff --git a/cmd/ctrlc/root/sync/salesforce/common/util.go b/cmd/ctrlc/root/sync/salesforce/common/util.go index 86264d2..f77be8c 100644 --- a/cmd/ctrlc/root/sync/salesforce/common/util.go +++ b/cmd/ctrlc/root/sync/salesforce/common/util.go @@ -29,36 +29,13 @@ func GetSalesforceSubdomain(domain string) string { return subdomain } -func ParseMetadataMappings(mappings []string) ([]string, map[string]string) { - fieldMap := make(map[string]bool) - lookupMap := make(map[string]string) // fieldName -> metadataKey - - for _, mapping := range mappings { - parts := strings.Split(mapping, "=") - if len(parts) == 2 { - metadataKey := parts[0] - fieldName := parts[1] - fieldMap[fieldName] = true - lookupMap[fieldName] = metadataKey - } - } - - fields := make([]string, 0, len(fieldMap)) - for field := range fieldMap { - fields = append(fields, field) - } - - return fields, lookupMap -} - func GetCustomFieldValue(obj interface{}, fieldName string) (string, bool) { objValue := reflect.ValueOf(obj) if objValue.Kind() == reflect.Ptr { objValue = objValue.Elem() } - customFields := objValue.FieldByName("CustomFields") - if customFields.IsValid() && customFields.Kind() == reflect.Map { + if customFields := objValue.FieldByName("CustomFields"); customFields.IsValid() && customFields.Kind() == reflect.Map { if value := customFields.MapIndex(reflect.ValueOf(fieldName)); value.IsValid() { return fmt.Sprintf("%v", value.Interface()), true } @@ -87,160 +64,153 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec if targetValue.Kind() != reflect.Slice { return fmt.Errorf("target must be a pointer to a slice") } - elementType := targetValue.Type().Elem() - fieldNames := []string{} + fieldNames := getFieldsToQuery(targetValue.Type().Elem(), additionalFields) + + if listAllFields { + if err := logAvailableFields(sf, objectName); err != nil { + return err + } + } + + return paginateQuery(ctx, sf, objectName, fieldNames, whereClause, limit, targetValue) +} + +// getFieldsToQuery extracts field names from struct tags and merges with additional fields +func getFieldsToQuery(elementType reflect.Type, additionalFields []string) []string { + // Use a map to automatically handle deduplication + fieldMap := make(map[string]bool) + for i := 0; i < elementType.NumField(); i++ { field := elementType.Field(i) jsonTag := field.Tag.Get("json") if jsonTag != "" && jsonTag != "-" { - fieldName := strings.Split(jsonTag, ",")[0] - if fieldName != "" { - fieldNames = append(fieldNames, fieldName) + if fieldName := strings.Split(jsonTag, ",")[0]; fieldName != "" { + fieldMap[fieldName] = true } } } for _, field := range additionalFields { - found := false - for _, existing := range fieldNames { - if existing == field { - found = true - break - } - } - if !found { - fieldNames = append(fieldNames, field) - } + fieldMap[field] = true } - if listAllFields { - describeResp, err := sf.DoRequest("GET", fmt.Sprintf("/sobjects/%s/describe", objectName), nil) - if err != nil { - return fmt.Errorf("failed to describe %s object: %w", objectName, err) - } - defer describeResp.Body.Close() + fields := make([]string, 0, len(fieldMap)) + for field := range fieldMap { + fields = append(fields, field) + } + return fields +} - var describeResult map[string]interface{} - if err := json.NewDecoder(describeResp.Body).Decode(&describeResult); err != nil { - return fmt.Errorf("failed to decode describe response: %w", err) - } +func logAvailableFields(sf *salesforce.Salesforce, objectName string) error { + resp, err := sf.DoRequest("GET", fmt.Sprintf("/sobjects/%s/describe", objectName), nil) + if err != nil { + return fmt.Errorf("failed to describe %s object: %w", objectName, err) + } + defer resp.Body.Close() - fields, ok := describeResult["fields"].([]interface{}) - if !ok { - return fmt.Errorf("unexpected describe response format") - } + var result map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return fmt.Errorf("failed to decode describe response: %w", err) + } - allFieldNames := []string{} - for _, field := range fields { - fieldMap, ok := field.(map[string]interface{}) - if !ok { - continue - } + fields, ok := result["fields"].([]interface{}) + if !ok { + return fmt.Errorf("unexpected describe response format") + } + + fieldNames := make([]string, 0, len(fields)) + for _, field := range fields { + if fieldMap, ok := field.(map[string]interface{}); ok { if name, ok := fieldMap["name"].(string); ok { - allFieldNames = append(allFieldNames, name) + fieldNames = append(fieldNames, name) } } - log.Info("Available fields", "object", objectName, "count", len(allFieldNames), "fields", allFieldNames) } - totalRetrieved := 0 - lastId := "" - batchSize := 2000 + log.Info("Available fields", "object", objectName, "count", len(fieldNames), "fields", fieldNames) + return nil +} - for { - fieldsClause := strings.Join(fieldNames, ", ") - baseQuery := fmt.Sprintf("SELECT %s FROM %s", fieldsClause, objectName) +func buildSOQL(objectName string, fields []string, whereClause string, lastId string, limit int) string { + query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(fields, ", "), objectName) - paginatedQuery := baseQuery - whereClauses := []string{} + conditions := []string{} + if whereClause != "" { + conditions = append(conditions, whereClause) + } + if lastId != "" { + conditions = append(conditions, fmt.Sprintf("Id > '%s'", lastId)) + } + if len(conditions) > 0 { + query += " WHERE " + strings.Join(conditions, " AND ") + } - if whereClause != "" { - whereClauses = append(whereClauses, whereClause) - } + query += " ORDER BY Id" - if lastId != "" { - whereClauses = append(whereClauses, fmt.Sprintf("Id > '%s'", lastId)) - } + if limit > 0 { + query += fmt.Sprintf(" LIMIT %d", limit) + } else { + query += " LIMIT 2000" // Default batch size + } - if len(whereClauses) > 0 { - paginatedQuery += " WHERE " + strings.Join(whereClauses, " AND ") - } - paginatedQuery += " ORDER BY Id" + return query +} - if limit > 0 && limit-totalRetrieved < batchSize { - paginatedQuery += fmt.Sprintf(" LIMIT %d", limit-totalRetrieved) - } else { - paginatedQuery += fmt.Sprintf(" LIMIT %d", batchSize) - } +func getRecordId(record reflect.Value) string { + if record.Kind() != reflect.Struct { + return "" + } + if idField := record.FieldByName("ID"); idField.IsValid() && idField.Kind() == reflect.String { + return idField.String() + } + if idField := record.FieldByName("Id"); idField.IsValid() && idField.Kind() == reflect.String { + return idField.String() + } + return "" +} - encodedQuery := url.QueryEscape(paginatedQuery) - queryURL := fmt.Sprintf("/query?q=%s", encodedQuery) +func paginateQuery(ctx context.Context, sf *salesforce.Salesforce, objectName string, fields []string, whereClause string, limit int, targetValue reflect.Value) error { + const batchSize = 2000 + totalRetrieved := 0 + lastId := "" - queryResp, err := sf.DoRequest("GET", queryURL, nil) - if err != nil { - return fmt.Errorf("failed to query %s: %w", objectName, err) + for { + batchLimit := batchSize + if limit > 0 && limit-totalRetrieved < batchSize { + batchLimit = limit - totalRetrieved } - body, err := io.ReadAll(queryResp.Body) + query := buildSOQL(objectName, fields, whereClause, lastId, batchLimit) + batch, err := executeQuery(sf, query, targetValue.Type()) if err != nil { - queryResp.Body.Close() - return fmt.Errorf("failed to read response body: %w", err) - } - queryResp.Body.Close() - - var queryResult struct { - TotalSize int `json:"totalSize"` - Done bool `json:"done"` - Records json.RawMessage `json:"records"` - NextRecordsUrl string `json:"nextRecordsUrl"` - } - - if err := json.Unmarshal(body, &queryResult); err != nil { - return fmt.Errorf("failed to unmarshal query response: %w", err) - } - - batchSlice := reflect.New(targetValue.Type()).Elem() - - if err := json.Unmarshal(queryResult.Records, batchSlice.Addr().Interface()); err != nil { - return fmt.Errorf("failed to unmarshal records: %w", err) + return fmt.Errorf("failed to query %s: %w", objectName, err) } - if batchSlice.Len() == 0 { + if batch.Len() == 0 { break } - for i := 0; i < batchSlice.Len(); i++ { - targetValue.Set(reflect.Append(targetValue, batchSlice.Index(i))) + for i := 0; i < batch.Len(); i++ { + targetValue.Set(reflect.Append(targetValue, batch.Index(i))) } - recordCount := batchSlice.Len() + recordCount := batch.Len() totalRetrieved += recordCount + // Get last ID for next page if recordCount > 0 { - lastRecord := batchSlice.Index(recordCount - 1) - if lastRecord.Kind() == reflect.Struct { - idField := lastRecord.FieldByName("ID") - if !idField.IsValid() { - idField = lastRecord.FieldByName("Id") - } - if idField.IsValid() && idField.Kind() == reflect.String { - lastId = idField.String() - } - } + lastId = getRecordId(batch.Index(recordCount - 1)) } log.Debug("Retrieved batch", "object", objectName, "batch_size", recordCount, "total", totalRetrieved) - if limit > 0 && totalRetrieved >= limit { - break - } - - if recordCount == 0 { + if (limit > 0 && totalRetrieved >= limit) || recordCount < batchLimit { break } } + // Trim to exact limit if needed if limit > 0 && targetValue.Len() > limit { targetValue.Set(targetValue.Slice(0, limit)) } @@ -248,6 +218,36 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec return nil } +// executeQuery executes a SOQL query and returns the unmarshaled records +func executeQuery(sf *salesforce.Salesforce, query string, targetType reflect.Type) (reflect.Value, error) { + encodedQuery := url.QueryEscape(query) + resp, err := sf.DoRequest("GET", fmt.Sprintf("/query?q=%s", encodedQuery), nil) + if err != nil { + return reflect.Value{}, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return reflect.Value{}, fmt.Errorf("failed to read response: %w", err) + } + + var result struct { + Records json.RawMessage `json:"records"` + } + if err := json.Unmarshal(body, &result); err != nil { + return reflect.Value{}, fmt.Errorf("failed to unmarshal response: %w", err) + } + + // Create a new slice of the target type to unmarshal into + batch := reflect.New(targetType).Elem() + if err := json.Unmarshal(result.Records, batch.Addr().Interface()); err != nil { + return reflect.Value{}, fmt.Errorf("failed to unmarshal records: %w", err) + } + + return batch, nil +} + func GetKnownFieldsFromStruct(structType reflect.Type) map[string]bool { knownFields := make(map[string]bool) diff --git a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go index cee9659..be9ffdd 100644 --- a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go +++ b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go @@ -84,7 +84,7 @@ func (o *Opportunity) UnmarshalJSON(data []byte) error { func NewSalesforceOpportunitiesCmd() *cobra.Command { var name string - var metadataMappings []string + var metadataMappings map[string]string var limit int var listAllFields bool var whereClause string @@ -155,7 +155,7 @@ func NewSalesforceOpportunitiesCmd() *cobra.Command { } cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") - cmd.Flags().StringArrayVar(&metadataMappings, "metadata", []string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") + cmd.Flags().StringToStringVar(&metadataMappings, "metadata", map[string]string{}, "Custom metadata mappings (format: metadata/key=SalesforceField)") cmd.Flags().IntVar(&limit, "limit", 0, "Maximum number of records to sync (0 = no limit)") cmd.Flags().BoolVar(&listAllFields, "list-all-fields", false, "List all available Salesforce fields in the logs") cmd.Flags().StringVar(&whereClause, "where", "", "SOQL WHERE clause to filter records (e.g., \"Amount > 100000\")") @@ -163,12 +163,12 @@ func NewSalesforceOpportunitiesCmd() *cobra.Command { return cmd } -// processOpportunities queries and transforms opportunities -func processOpportunities(ctx context.Context, sf *salesforce.Salesforce, metadataMappings []string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { - // Parse metadata mappings to get field names for query - additionalFields, mappingLookup := common.ParseMetadataMappings(metadataMappings) +func processOpportunities(ctx context.Context, sf *salesforce.Salesforce, metadataMappings map[string]string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { + additionalFields := make([]string, 0, len(metadataMappings)) + for _, fieldName := range metadataMappings { + additionalFields = append(additionalFields, fieldName) + } - // Query Salesforce for opportunities var opportunities []Opportunity err := common.QuerySalesforceObject(ctx, sf, "Opportunity", limit, listAllFields, &opportunities, additionalFields, whereClause) if err != nil { @@ -177,18 +177,16 @@ func processOpportunities(ctx context.Context, sf *salesforce.Salesforce, metada log.Info("Found Salesforce opportunities", "count", len(opportunities)) - // Transform opportunities to Ctrlplane resources resources := []api.CreateResource{} for _, opp := range opportunities { - resource := transformOpportunityToResource(opp, mappingLookup) + resource := transformOpportunityToResource(opp, metadataMappings) resources = append(resources, resource) } return resources, nil } -func transformOpportunityToResource(opportunity Opportunity, mappingLookup map[string]string) api.CreateResource { - // Format close date +func transformOpportunityToResource(opportunity Opportunity, metadataMappings map[string]string) api.CreateResource { var closeDateFormatted string if opportunity.CloseDate != "" { if t, err := time.Parse("2006-01-02", opportunity.CloseDate); err == nil { @@ -219,8 +217,7 @@ func transformOpportunityToResource(opportunity Opportunity, mappingLookup map[s "opportunity/last-modified": opportunity.LastModifiedDate, } - // Apply custom metadata mappings - for fieldName, metadataKey := range mappingLookup { + for metadataKey, fieldName := range metadataMappings { if value, found := common.GetCustomFieldValue(opportunity, fieldName); found { metadata[metadataKey] = value } From 1bfc1afc9f9e304e37f45a54592ded03913565fb Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Wed, 16 Jul 2025 13:54:21 -0700 Subject: [PATCH 5/8] refactor --- cmd/ctrlc/root/sync/salesforce/README.md | 363 +++--------------- .../root/sync/salesforce/accounts/accounts.go | 178 +++------ cmd/ctrlc/root/sync/salesforce/common/util.go | 155 +++----- .../salesforce/opportunities/opportunities.go | 199 ++++------ cmd/ctrlc/root/sync/salesforce/salesforce.go | 34 +- 5 files changed, 262 insertions(+), 667 deletions(-) diff --git a/cmd/ctrlc/root/sync/salesforce/README.md b/cmd/ctrlc/root/sync/salesforce/README.md index 4c5409e..8467e4b 100644 --- a/cmd/ctrlc/root/sync/salesforce/README.md +++ b/cmd/ctrlc/root/sync/salesforce/README.md @@ -1,173 +1,62 @@ # Salesforce Sync -This package provides functionality to sync Salesforce CRM data into Ctrlplane as resources. +Sync Salesforce CRM data (Accounts, Opportunities) into Ctrlplane as resources. -## Usage - -### Prerequisites - -You need Salesforce OAuth2 credentials: -- **Domain**: Your Salesforce instance URL (e.g., `https://mycompany.my.salesforce.com`) -- **Consumer Key**: From your Salesforce Connected App -- **Consumer Secret**: From your Salesforce Connected App - -### Setting up Salesforce Connected App - -1. Go to Setup → Apps → App Manager -2. Click "New Connected App" -3. Fill in the required fields -4. Enable OAuth Settings -5. Add OAuth Scopes: - - `api` - Access and manage your data - - `refresh_token` - Perform requests on your behalf at any time -6. Save and note the Consumer Key and Consumer Secret - -### Authentication - -The Salesforce credentials are configured at the parent `salesforce` command level and apply to all subcommands (accounts, opportunities, etc.). - -You can provide credentials via environment variables: +## Quick Start ```bash +# Set credentials (via environment or flags) export SALESFORCE_DOMAIN="https://mycompany.my.salesforce.com" -export SALESFORCE_CONSUMER_KEY="your-consumer-key" -export SALESFORCE_CONSUMER_SECRET="your-consumer-secret" -``` - -Or via command-line flags: - -```bash -ctrlc sync salesforce accounts \ - --salesforce-domain "https://mycompany.my.salesforce.com" \ - --salesforce-consumer-key "your-consumer-key" \ - --salesforce-consumer-secret "your-consumer-secret" -``` - -### Command-Line Flags - -#### Global Salesforce Flags (apply to all subcommands) - -| Flag | Description | Default | -|------|-------------|---------| -| `--salesforce-domain` | Salesforce instance URL | `$SALESFORCE_DOMAIN` | -| `--salesforce-consumer-key` | OAuth2 consumer key | `$SALESFORCE_CONSUMER_KEY` | -| `--salesforce-consumer-secret` | OAuth2 consumer secret | `$SALESFORCE_CONSUMER_SECRET` | +export SALESFORCE_CONSUMER_KEY="your-key" +export SALESFORCE_CONSUMER_SECRET="your-secret" -#### Subcommand Flags - -Both `accounts` and `opportunities` commands support the following flags: - -| Flag | Description | Default | -|------|-------------|---------| -| `--provider`, `-p` | Resource provider name | Auto-generated from domain (e.g., `wandb-salesforce-accounts`) | -| `--metadata` | Custom metadata mappings (can be used multiple times) | Built-in defaults | -| `--where` | SOQL WHERE clause to filter records | None (syncs all records) | -| `--limit` | Maximum number of records to sync | 0 (no limit) | -| `--list-all-fields` | Log all available Salesforce fields | false | - -### Syncing Accounts - -```bash -# Sync all Salesforce accounts +# Sync all accounts ctrlc sync salesforce accounts -# Sync accounts with a filter (e.g., only accounts with Customer Health populated) -ctrlc sync salesforce accounts --where="Customer_Health__c != null" - -# Sync accounts with complex filters -ctrlc sync salesforce accounts --where="Type = 'Customer' AND AnnualRevenue > 1000000" - -# Sync accounts and list all available fields in logs -ctrlc sync salesforce accounts --list-all-fields +# Sync opportunities with filters +ctrlc sync salesforce opportunities --where="IsWon = true AND Amount > 50000" -# Sync with custom provider name -ctrlc sync salesforce accounts --provider my-salesforce-accounts - -# Limit the number of records to sync -ctrlc sync salesforce accounts --limit 500 - -# Combine filters with metadata mappings -ctrlc sync salesforce accounts \ - --where="Industry = 'Technology'" \ - --metadata="account/revenue=AnnualRevenue" -``` - -#### Custom Field Mappings - -You can map any Salesforce field (including custom fields) to Ctrlplane metadata: - -```bash -# Map standard and custom fields to metadata +# Map custom fields to metadata ctrlc sync salesforce accounts \ --metadata="account/tier=Tier__c" \ - --metadata="account/region=Region__c" \ - --metadata="account/annual-revenue=AnnualRevenue" \ - --metadata="account/health=Customer_Health__c" \ - --metadata="account/contract-value=Contract_Value__c" + --metadata="account/health=Customer_Health__c" ``` -**Key Points about Metadata Mappings**: -- Format: `metadata-key=SalesforceFieldName` -- The left side is the metadata key in Ctrlplane (e.g., `account/tier`) -- The right side is the exact Salesforce field name (e.g., `Tier__c`) -- Custom fields in Salesforce typically end with `__c` -- All values are stored as strings in metadata -- Use `--list-all-fields` to discover available field names +## Authentication -### Default Provider Naming +Requires Salesforce OAuth2 credentials from a Connected App with `api` and `refresh_token` scopes. -If you don't specify a `--provider` name, the system automatically generates one based on your Salesforce domain: -- `https://wandb.my.salesforce.com` → `wandb-salesforce-accounts` -- `https://acme.my.salesforce.com` → `acme-salesforce-accounts` -- `https://mycompany.my.salesforce.com` → `mycompany-salesforce-accounts` - -### Syncing Opportunities - -```bash -# Sync all Salesforce opportunities -ctrlc sync salesforce opportunities - -# Sync only open opportunities -ctrlc sync salesforce opportunities --where="IsClosed = false" +Credentials can be provided via: +- Environment variables: `SALESFORCE_DOMAIN`, `SALESFORCE_CONSUMER_KEY`, `SALESFORCE_CONSUMER_SECRET` +- Command flags: `--salesforce-domain`, `--salesforce-consumer-key`, `--salesforce-consumer-secret` -# Sync opportunities with complex filters -ctrlc sync salesforce opportunities --where="Amount > 50000 AND StageName != 'Closed Lost'" +## Common Flags -# Sync opportunities and list all available fields in logs -ctrlc sync salesforce opportunities --list-all-fields - -# Sync with custom provider name -ctrlc sync salesforce opportunities --provider my-salesforce-opportunities - -# Limit the number of records to sync -ctrlc sync salesforce opportunities --limit 500 - -# Combine filters with metadata mappings -ctrlc sync salesforce opportunities \ - --where="Amount > 100000" \ - --metadata="opportunity/probability=Probability" -``` +| Flag | Description | Default | +|------|-------------|---------| +| `--provider`, `-p` | Resource provider name | Auto-generated from domain | +| `--metadata` | Map Salesforce fields to metadata | Built-in defaults | +| `--where` | SOQL WHERE clause filter | None | +| `--limit` | Maximum records to sync | 0 (no limit) | +| `--list-all-fields` | Log available Salesforce fields | false | -#### Custom Field Mappings +## Metadata Mappings -Just like accounts, you can map any Salesforce opportunity field (including custom fields) to Ctrlplane metadata: +Map any Salesforce field (including custom fields) to Ctrlplane metadata: ```bash -# Map standard and custom fields to metadata -ctrlc sync salesforce opportunities \ - --metadata="opportunity/type=Type__c" \ - --metadata="opportunity/expected-revenue=ExpectedRevenue" \ - --metadata="opportunity/lead-source=LeadSource" \ - --metadata="opportunity/next-step=NextStep" \ - --metadata="opportunity/use-case=Use_Case__c" +# Format: metadata-key=SalesforceFieldName +--metadata="account/tier=Tier__c" +--metadata="opportunity/stage-custom=Custom_Stage__c" ``` -## Resource Schema +- Custom fields typically end with `__c` +- Use `--list-all-fields` to discover available fields +- All metadata values are stored as strings -### Salesforce Account Resource - -Resources are created with the following structure: +## Resource Examples +### Account Resource ```json { "version": "ctrlplane.dev/crm/account/v1", @@ -176,205 +65,73 @@ Resources are created with the following structure: "identifier": "001XX000003DHPh", "config": { "name": "Acme Corporation", - "industry": "Technology", - "id": "001XX000003DHPh", "type": "Customer", - "phone": "+1-555-0123", - "website": "https://acme.com", "salesforceAccount": { "recordId": "001XX000003DHPh", "ownerId": "005XX000001SvogAAC", - "parentId": "", - "type": "Customer", - "accountSource": "Web", - "numberOfEmployees": 5000, - "description": "Major technology customer", - "billingAddress": { - "street": "123 Main St", - "city": "San Francisco", - "state": "CA", - "postalCode": "94105", - "country": "USA", - "latitude": 37.7749, - "longitude": -122.4194 - }, - "shippingAddress": { - "street": "123 Main St", - "city": "San Francisco", - "state": "CA", - "postalCode": "94105", - "country": "USA", - "latitude": 37.7749, - "longitude": -122.4194 - }, - "createdDate": "2023-01-15T10:30:00Z", - "lastModifiedDate": "2024-01-20T15:45:00Z", - "isDeleted": false, - "photoUrl": "https://..." + // ... address, dates, etc. } }, "metadata": { - "ctrlplane/external-id": "001XX000003DHPh", "account/id": "001XX000003DHPh", - "account/owner-id": "005XX000001SvogAAC", - "account/industry": "Technology", - "account/billing-city": "San Francisco", - "account/billing-state": "CA", - "account/billing-country": "USA", - "account/website": "https://acme.com", - "account/phone": "+1-555-0123", "account/type": "Customer", - "account/source": "Web", - "account/shipping-city": "San Francisco", - "account/parent-id": "", - "account/employees": "5000", - // Custom fields added via --metadata mappings - "account/tier": "Enterprise", - "account/health": "Green" + // Custom fields from --metadata + "account/tier": "Enterprise" } } ``` -### Salesforce Opportunity Resource - +### Opportunity Resource ```json { - "version": "ctrlplane.dev/crm/opportunity/v1", + "version": "ctrlplane.dev/crm/opportunity/v1", "kind": "SalesforceOpportunity", - "name": "Acme Corp - Enterprise Deal", + "name": "Enterprise Deal", "identifier": "006XX000003DHPh", "config": { - "name": "Acme Corp - Enterprise Deal", "amount": 250000, - "stage": "Negotiation/Review", - "id": "006XX000003DHPh", - "probability": 75, - "isClosed": false, - "isWon": false, + "stage": "Negotiation", "salesforceOpportunity": { "recordId": "006XX000003DHPh", "accountId": "001XX000003DHPh", - "ownerId": "005XX000001SvogAAC", - "type": "New Business", - "leadSource": "Partner Referral", - "closeDate": "2024-12-31T00:00:00Z", - "forecastCategory": "Commit", - "description": "Enterprise license upgrade", - "nextStep": "Legal review", - "hasOpenActivity": true, - "createdDate": "2024-01-15T10:30:00Z", - "lastModifiedDate": "2024-02-20T15:45:00Z", - "lastActivityDate": "2024-02-19T00:00:00Z", - "fiscalQuarter": 4, - "fiscalYear": 2024 + // ... dates, fiscal info, etc. } }, "metadata": { - "ctrlplane/external-id": "006XX000003DHPh", - "opportunity/id": "006XX000003DHPh", - "opportunity/account-id": "001XX000003DHPh", - "opportunity/owner-id": "005XX000001SvogAAC", - "opportunity/stage": "Negotiation/Review", "opportunity/amount": "250000", - "opportunity/probability": "75", - "opportunity/close-date": "2024-12-31", - "opportunity/type": "New Business", - "opportunity/lead-source": "Partner Referral", - "opportunity/is-closed": "false", - "opportunity/is-won": "false", - // Custom fields added via --metadata mappings - "opportunity/use-case": "Platform Migration", - "opportunity/competition": "Competitor X" + "opportunity/stage": "Negotiation" } } ``` -## Implementation Details - -This integration uses the [go-salesforce](https://github.com/k-capehart/go-salesforce) library for OAuth2 authentication and SOQL queries. +## Advanced Usage -### Features +### Filtering with SOQL -- **Dynamic Field Discovery**: Automatically discovers and fetches all available fields (standard and custom) from Salesforce objects -- **Custom Field Mappings**: Map any Salesforce field to Ctrlplane metadata using `--metadata` flags -- **Flexible Filtering**: Use SOQL WHERE clauses with the `--where` flag to filter records -- **Smart Field Capture**: Automatically captures any custom Salesforce fields (ending in `__c`) not defined in the struct -- **Automatic Pagination**: Handles large datasets efficiently with ID-based pagination -- **Subdomain-based Naming**: Automatically generates provider names from your Salesforce subdomain -- **Required Field Validation**: Uses Cobra's `MarkFlagRequired` for proper validation - -### Core Architecture - -The sync implementation follows a clean, modular architecture: - -1. **Parse Metadata Mappings**: `ParseMetadataMappings` parses the `--metadata` flags once, returning: - - Field names to include in the SOQL query - - A lookup map for transforming fields to metadata keys - -2. **Query Salesforce**: `QuerySalesforceObject` performs the actual SOQL query with: - - Dynamic field selection based on struct tags and metadata mappings - - Automatic pagination handling - - Optional field listing for discovery - -3. **Transform to Resources**: Each object is transformed into a Ctrlplane resource with: - - Standard metadata mappings - - Custom field mappings from the `--metadata` flags - - Proper type conversion (all metadata values are strings) - -4. **Upload to Ctrlplane**: `UpsertToCtrlplane` handles the resource upload - -### Handling Custom Fields - -Salesforce custom fields (typically ending in `__c`) are handled through: - -1. **Automatic Capture**: The `UnmarshalJSON` method captures any fields not in the struct into a `CustomFields` map -2. **Metadata Mapping**: Use `--metadata` flags to map these fields to Ctrlplane metadata -3. **Field Discovery**: Use `--list-all-fields` to see all available fields in your Salesforce instance - -Example workflow: ```bash -# 1. Discover available fields -ctrlc sync salesforce accounts --list-all-fields --limit 1 - -# 2. Map the custom fields you need +# Complex account filters ctrlc sync salesforce accounts \ - --metadata="account/tier=Customer_Tier__c" \ - --metadata="account/segment=Market_Segment__c" \ - --metadata="account/arr=Annual_Recurring_Revenue__c" -``` - -### Shared Utilities + --where="Type = 'Customer' AND AnnualRevenue > 1000000" -The `common/` package provides reusable functions for all Salesforce object syncs: - -- **`InitSalesforceClient`**: Sets up OAuth2 authentication -- **`ParseMetadataMappings`**: Parses `--metadata` flags into field lists and lookup maps -- **`QuerySalesforceObject`**: Generic SOQL query with pagination -- **`GetCustomFieldValue`**: Gets any field value from struct (standard or custom) -- **`UnmarshalWithCustomFields`**: Captures unknown fields from Salesforce -- **`GetKnownFieldsFromStruct`**: Extracts field names from struct tags -- **`GetSalesforceSubdomain`**: Extracts subdomain for default provider naming -- **`UpsertToCtrlplane`**: Handles resource upload to Ctrlplane +# Filter opportunities by custom fields +ctrlc sync salesforce opportunities \ + --where="Custom_Field__c != null AND Stage = 'Closed Won'" +``` ### Pagination -Records are fetched efficiently using Salesforce best practices: -- ID-based pagination (avoids OFFSET limitations) -- Configurable batch size (default: 2000 records) -- Ordered by ID for consistent results +- Automatically handles large datasets with ID-based pagination +- Fetches up to 1000 records per API call - Use `--limit` to restrict total records synced -### Adding New Salesforce Objects +### Default Provider Names -To add support for a new Salesforce object (e.g., Leads): +If no `--provider` is specified, names are auto-generated from your Salesforce subdomain: +- `https://acme.my.salesforce.com` → `acme-salesforce-accounts` -1. Create a new struct with JSON tags matching Salesforce field names -2. Include a `CustomFields map[string]interface{}` field -3. Implement `UnmarshalJSON` to capture custom fields -4. Create a command that: - - Uses `ParseMetadataMappings` for field mappings - - Calls `QuerySalesforceObject` for data retrieval - - Transforms objects to Ctrlplane resources - - Uses `UpsertToCtrlplane` for upload +## Implementation Notes -The shared utilities handle most of the complexity, making new object support straightforward. \ No newline at end of file +- Uses `map[string]any` for Salesforce's dynamic schema +- Null values are omitted from resources +- Numbers and booleans are preserved in config, converted to strings in metadata +- Dates are formatted to RFC3339 where applicable \ No newline at end of file diff --git a/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go index 897c1c8..2fec776 100644 --- a/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go +++ b/cmd/ctrlc/root/sync/salesforce/accounts/accounts.go @@ -3,8 +3,6 @@ package accounts import ( "context" "fmt" - "reflect" - "strconv" "github.com/MakeNowJust/heredoc/v2" "github.com/charmbracelet/log" @@ -15,78 +13,6 @@ import ( "github.com/spf13/viper" ) -type Account struct { - ID string `json:"Id"` // this is the globally unique identifier for the account - IsDeleted bool `json:"IsDeleted"` - MasterRecordId string `json:"MasterRecordId"` - Name string `json:"Name"` - Type string `json:"Type"` - ParentId string `json:"ParentId"` - BillingStreet string `json:"BillingStreet"` - BillingCity string `json:"BillingCity"` - BillingState string `json:"BillingState"` - BillingPostalCode string `json:"BillingPostalCode"` - BillingCountry string `json:"BillingCountry"` - BillingLatitude float64 `json:"BillingLatitude"` - BillingLongitude float64 `json:"BillingLongitude"` - BillingGeocodeAccuracy string `json:"BillingGeocodeAccuracy"` - BillingAddress interface{} `json:"BillingAddress"` - ShippingStreet string `json:"ShippingStreet"` - ShippingCity string `json:"ShippingCity"` - ShippingState string `json:"ShippingState"` - ShippingPostalCode string `json:"ShippingPostalCode"` - ShippingCountry string `json:"ShippingCountry"` - ShippingLatitude float64 `json:"ShippingLatitude"` - ShippingLongitude float64 `json:"ShippingLongitude"` - ShippingGeocodeAccuracy string `json:"ShippingGeocodeAccuracy"` - ShippingAddress interface{} `json:"ShippingAddress"` - Phone string `json:"Phone"` - Website string `json:"Website"` - PhotoUrl string `json:"PhotoUrl"` - Industry string `json:"Industry"` - NumberOfEmployees int `json:"NumberOfEmployees"` - Description string `json:"Description"` - OwnerId string `json:"OwnerId"` - CreatedDate string `json:"CreatedDate"` - CreatedById string `json:"CreatedById"` - LastModifiedDate string `json:"LastModifiedDate"` - LastModifiedById string `json:"LastModifiedById"` - SystemModstamp string `json:"SystemModstamp"` - LastActivityDate string `json:"LastActivityDate"` - LastViewedDate string `json:"LastViewedDate"` - LastReferencedDate string `json:"LastReferencedDate"` - Jigsaw string `json:"Jigsaw"` - JigsawCompanyId string `json:"JigsawCompanyId"` - AccountSource string `json:"AccountSource"` - SicDesc string `json:"SicDesc"` - IsPriorityRecord bool `json:"IsPriorityRecord"` - - // CustomFields holds any additional fields not defined in the struct - // This allows handling of custom Salesforce fields like Tier__c with - // the --metadata flag. - CustomFields map[string]interface{} `json:"-"` -} - -func (a *Account) UnmarshalJSON(data []byte) error { - type Alias Account - aux := &struct { - *Alias - }{ - Alias: (*Alias)(a), - } - - knownFields := common.GetKnownFieldsFromStruct(reflect.TypeOf(Account{})) - - customFields, err := common.UnmarshalWithCustomFields(data, aux, knownFields) - if err != nil { - return err - } - - a.CustomFields = customFields - - return nil -} - func NewSalesforceAccountsCmd() *cobra.Command { var name string var metadataMappings map[string]string @@ -176,7 +102,7 @@ func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMap additionalFields = append(additionalFields, fieldName) } - var accounts []Account + var accounts []map[string]any err := common.QuerySalesforceObject(ctx, sf, "Account", limit, listAllFields, &accounts, additionalFields, whereClause) if err != nil { return nil, err @@ -193,76 +119,74 @@ func processAccounts(ctx context.Context, sf *salesforce.Salesforce, metadataMap return resources, nil } -func transformAccountToResource(account Account, metadataMappings map[string]string) api.CreateResource { - metadata := map[string]string{ - "ctrlplane/external-id": account.ID, - "account/id": account.ID, - "account/owner-id": account.OwnerId, - "account/industry": account.Industry, - "account/billing-city": account.BillingCity, - "account/billing-state": account.BillingState, - "account/billing-country": account.BillingCountry, - "account/website": account.Website, - "account/phone": account.Phone, - "account/type": account.Type, - "account/source": account.AccountSource, - "account/shipping-city": account.ShippingCity, - "account/parent-id": account.ParentId, - "account/employees": strconv.Itoa(account.NumberOfEmployees), - } +func transformAccountToResource(account map[string]any, metadataMappings map[string]string) api.CreateResource { + metadata := map[string]string{} + common.AddToMetadata(metadata, "account/id", account["Id"]) + common.AddToMetadata(metadata, "account/owner-id", account["OwnerId"]) + common.AddToMetadata(metadata, "account/industry", account["Industry"]) + common.AddToMetadata(metadata, "account/billing-city", account["BillingCity"]) + common.AddToMetadata(metadata, "account/billing-state", account["BillingState"]) + common.AddToMetadata(metadata, "account/billing-country", account["BillingCountry"]) + common.AddToMetadata(metadata, "account/website", account["Website"]) + common.AddToMetadata(metadata, "account/phone", account["Phone"]) + common.AddToMetadata(metadata, "account/type", account["Type"]) + common.AddToMetadata(metadata, "account/source", account["AccountSource"]) + common.AddToMetadata(metadata, "account/shipping-city", account["ShippingCity"]) + common.AddToMetadata(metadata, "account/parent-id", account["ParentId"]) + common.AddToMetadata(metadata, "account/employees", account["NumberOfEmployees"]) for metadataKey, fieldName := range metadataMappings { - if value, found := common.GetCustomFieldValue(account, fieldName); found { - metadata[metadataKey] = value + if value, exists := account[fieldName]; exists { + common.AddToMetadata(metadata, metadataKey, value) } } config := map[string]interface{}{ - "name": account.Name, - "industry": account.Industry, - "id": account.ID, - "type": account.Type, - "phone": account.Phone, - "website": account.Website, + "name": fmt.Sprintf("%v", account["Name"]), + "industry": fmt.Sprintf("%v", account["Industry"]), + "id": fmt.Sprintf("%v", account["Id"]), + "type": fmt.Sprintf("%v", account["Type"]), + "phone": fmt.Sprintf("%v", account["Phone"]), + "website": fmt.Sprintf("%v", account["Website"]), "salesforceAccount": map[string]interface{}{ - "recordId": account.ID, - "ownerId": account.OwnerId, - "parentId": account.ParentId, - "type": account.Type, - "accountSource": account.AccountSource, - "numberOfEmployees": account.NumberOfEmployees, - "description": account.Description, + "recordId": fmt.Sprintf("%v", account["Id"]), + "ownerId": fmt.Sprintf("%v", account["OwnerId"]), + "parentId": fmt.Sprintf("%v", account["ParentId"]), + "type": fmt.Sprintf("%v", account["Type"]), + "accountSource": fmt.Sprintf("%v", account["AccountSource"]), + "numberOfEmployees": account["NumberOfEmployees"], + "description": fmt.Sprintf("%v", account["Description"]), "billingAddress": map[string]interface{}{ - "street": account.BillingStreet, - "city": account.BillingCity, - "state": account.BillingState, - "postalCode": account.BillingPostalCode, - "country": account.BillingCountry, - "latitude": account.BillingLatitude, - "longitude": account.BillingLongitude, + "street": fmt.Sprintf("%v", account["BillingStreet"]), + "city": fmt.Sprintf("%v", account["BillingCity"]), + "state": fmt.Sprintf("%v", account["BillingState"]), + "postalCode": fmt.Sprintf("%v", account["BillingPostalCode"]), + "country": fmt.Sprintf("%v", account["BillingCountry"]), + "latitude": account["BillingLatitude"], + "longitude": account["BillingLongitude"], }, "shippingAddress": map[string]interface{}{ - "street": account.ShippingStreet, - "city": account.ShippingCity, - "state": account.ShippingState, - "postalCode": account.ShippingPostalCode, - "country": account.ShippingCountry, - "latitude": account.ShippingLatitude, - "longitude": account.ShippingLongitude, + "street": fmt.Sprintf("%v", account["ShippingStreet"]), + "city": fmt.Sprintf("%v", account["ShippingCity"]), + "state": fmt.Sprintf("%v", account["ShippingState"]), + "postalCode": fmt.Sprintf("%v", account["ShippingPostalCode"]), + "country": fmt.Sprintf("%v", account["ShippingCountry"]), + "latitude": account["ShippingLatitude"], + "longitude": account["ShippingLongitude"], }, - "createdDate": account.CreatedDate, - "lastModifiedDate": account.LastModifiedDate, - "isDeleted": account.IsDeleted, - "photoUrl": account.PhotoUrl, + "createdDate": fmt.Sprintf("%v", account["CreatedDate"]), + "lastModifiedDate": fmt.Sprintf("%v", account["LastModifiedDate"]), + "isDeleted": account["IsDeleted"], + "photoUrl": fmt.Sprintf("%v", account["PhotoUrl"]), }, } return api.CreateResource{ Version: "ctrlplane.dev/crm/account/v1", Kind: "SalesforceAccount", - Name: account.Name, - Identifier: account.ID, + Name: fmt.Sprintf("%v", account["Name"]), + Identifier: fmt.Sprintf("%v", account["Id"]), Config: config, Metadata: metadata, } diff --git a/cmd/ctrlc/root/sync/salesforce/common/util.go b/cmd/ctrlc/root/sync/salesforce/common/util.go index f77be8c..83a87ef 100644 --- a/cmd/ctrlc/root/sync/salesforce/common/util.go +++ b/cmd/ctrlc/root/sync/salesforce/common/util.go @@ -29,35 +29,6 @@ func GetSalesforceSubdomain(domain string) string { return subdomain } -func GetCustomFieldValue(obj interface{}, fieldName string) (string, bool) { - objValue := reflect.ValueOf(obj) - if objValue.Kind() == reflect.Ptr { - objValue = objValue.Elem() - } - - if customFields := objValue.FieldByName("CustomFields"); customFields.IsValid() && customFields.Kind() == reflect.Map { - if value := customFields.MapIndex(reflect.ValueOf(fieldName)); value.IsValid() { - return fmt.Sprintf("%v", value.Interface()), true - } - } - - objType := objValue.Type() - for i := 0; i < objType.NumField(); i++ { - field := objType.Field(i) - if jsonTag := field.Tag.Get("json"); jsonTag != "" { - tagName := strings.Split(jsonTag, ",")[0] - if tagName == fieldName { - fieldValue := objValue.Field(i) - if fieldValue.IsValid() && fieldValue.CanInterface() { - return fmt.Sprintf("%v", fieldValue.Interface()), true - } - } - } - } - - return "", false -} - // QuerySalesforceObject performs a generic query on any Salesforce object with pagination support func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objectName string, limit int, listAllFields bool, target interface{}, additionalFields []string, whereClause string) error { targetValue := reflect.ValueOf(target).Elem() @@ -65,41 +36,51 @@ func QuerySalesforceObject(ctx context.Context, sf *salesforce.Salesforce, objec return fmt.Errorf("target must be a pointer to a slice") } - fieldNames := getFieldsToQuery(targetValue.Type().Elem(), additionalFields) + fieldMap := make(map[string]bool) - if listAllFields { - if err := logAvailableFields(sf, objectName); err != nil { - return err + var standardFields []string + switch objectName { + case "Account": + standardFields = []string{ + "Id", "Name", "Type", "Industry", "Website", "Phone", + "BillingStreet", "BillingCity", "BillingState", "BillingPostalCode", "BillingCountry", + "BillingLatitude", "BillingLongitude", + "ShippingStreet", "ShippingCity", "ShippingState", "ShippingPostalCode", "ShippingCountry", + "ShippingLatitude", "ShippingLongitude", + "NumberOfEmployees", "Description", "OwnerId", "ParentId", "AccountSource", + "CreatedDate", "LastModifiedDate", "IsDeleted", "PhotoUrl", } + case "Opportunity": + standardFields = []string{ + "Id", "Name", "Amount", "StageName", "CloseDate", "AccountId", + "Probability", "Type", "NextStep", "LeadSource", "IsClosed", "IsWon", + "ForecastCategory", "Description", "OwnerId", "ContactId", "CampaignId", + "HasOpenActivity", "CreatedDate", "LastModifiedDate", "LastActivityDate", + } + default: + standardFields = []string{"Id", "Name", "CreatedDate", "LastModifiedDate"} } - return paginateQuery(ctx, sf, objectName, fieldNames, whereClause, limit, targetValue) -} - -// getFieldsToQuery extracts field names from struct tags and merges with additional fields -func getFieldsToQuery(elementType reflect.Type, additionalFields []string) []string { - // Use a map to automatically handle deduplication - fieldMap := make(map[string]bool) - - for i := 0; i < elementType.NumField(); i++ { - field := elementType.Field(i) - jsonTag := field.Tag.Get("json") - if jsonTag != "" && jsonTag != "-" { - if fieldName := strings.Split(jsonTag, ",")[0]; fieldName != "" { - fieldMap[fieldName] = true - } - } + for _, field := range standardFields { + fieldMap[field] = true } for _, field := range additionalFields { fieldMap[field] = true } - fields := make([]string, 0, len(fieldMap)) + fieldNames := make([]string, 0, len(fieldMap)) for field := range fieldMap { - fields = append(fields, field) + fieldNames = append(fieldNames, field) + } + + if listAllFields { + if err := logAvailableFields(sf, objectName); err != nil { + return err + } } - return fields + + return paginateQuery(ctx, sf, objectName, fieldNames, whereClause, limit, targetValue) } func logAvailableFields(sf *salesforce.Salesforce, objectName string) error { @@ -132,6 +113,7 @@ func logAvailableFields(sf *salesforce.Salesforce, objectName string) error { return nil } +// buildSOQL constructs a SOQL query with pagination func buildSOQL(objectName string, fields []string, whereClause string, lastId string, limit int) string { query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(fields, ", "), objectName) @@ -150,28 +132,27 @@ func buildSOQL(objectName string, fields []string, whereClause string, lastId st if limit > 0 { query += fmt.Sprintf(" LIMIT %d", limit) - } else { - query += " LIMIT 2000" // Default batch size } return query } func getRecordId(record reflect.Value) string { - if record.Kind() != reflect.Struct { - return "" - } - if idField := record.FieldByName("ID"); idField.IsValid() && idField.Kind() == reflect.String { - return idField.String() - } - if idField := record.FieldByName("Id"); idField.IsValid() && idField.Kind() == reflect.String { - return idField.String() + if record.Kind() == reflect.Map && record.Type().Key().Kind() == reflect.String { + for _, key := range record.MapKeys() { + if key.String() == "Id" { + idValue := record.MapIndex(key) + if idValue.IsValid() && idValue.CanInterface() { + return fmt.Sprintf("%v", idValue.Interface()) + } + } + } } return "" } func paginateQuery(ctx context.Context, sf *salesforce.Salesforce, objectName string, fields []string, whereClause string, limit int, targetValue reflect.Value) error { - const batchSize = 2000 + const batchSize = 200 totalRetrieved := 0 lastId := "" @@ -198,7 +179,6 @@ func paginateQuery(ctx context.Context, sf *salesforce.Salesforce, objectName st recordCount := batch.Len() totalRetrieved += recordCount - // Get last ID for next page if recordCount > 0 { lastId = getRecordId(batch.Index(recordCount - 1)) } @@ -210,7 +190,6 @@ func paginateQuery(ctx context.Context, sf *salesforce.Salesforce, objectName st } } - // Trim to exact limit if needed if limit > 0 && targetValue.Len() > limit { targetValue.Set(targetValue.Slice(0, limit)) } @@ -239,7 +218,6 @@ func executeQuery(sf *salesforce.Salesforce, query string, targetType reflect.Ty return reflect.Value{}, fmt.Errorf("failed to unmarshal response: %w", err) } - // Create a new slice of the target type to unmarshal into batch := reflect.New(targetType).Elem() if err := json.Unmarshal(result.Records, batch.Addr().Interface()); err != nil { return reflect.Value{}, fmt.Errorf("failed to unmarshal records: %w", err) @@ -248,48 +226,13 @@ func executeQuery(sf *salesforce.Salesforce, query string, targetType reflect.Ty return batch, nil } -func GetKnownFieldsFromStruct(structType reflect.Type) map[string]bool { - knownFields := make(map[string]bool) - - if structType.Kind() == reflect.Ptr { - structType = structType.Elem() - } - - for i := 0; i < structType.NumField(); i++ { - field := structType.Field(i) - jsonTag := field.Tag.Get("json") - - if jsonTag == "" || jsonTag == "-" { - continue - } - - fieldName := strings.Split(jsonTag, ",")[0] - if fieldName != "" { - knownFields[fieldName] = true +func AddToMetadata(metadata map[string]string, key string, value any) { + if value != nil { + strVal := fmt.Sprintf("%v", value) + if strVal != "" && strVal != "" { + metadata[key] = strVal } } - - return knownFields -} - -func UnmarshalWithCustomFields(data []byte, target interface{}, knownFields map[string]bool) (map[string]interface{}, error) { - if err := json.Unmarshal(data, target); err != nil { - return nil, err - } - - var allFields map[string]interface{} - if err := json.Unmarshal(data, &allFields); err != nil { - return nil, err - } - - customFields := make(map[string]interface{}) - for fieldName, value := range allFields { - if !knownFields[fieldName] { - customFields[fieldName] = value - } - } - - return customFields, nil } func UpsertToCtrlplane(ctx context.Context, resources []api.CreateResource, providerName string) error { diff --git a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go index be9ffdd..75dd0f5 100644 --- a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go +++ b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go @@ -3,8 +3,6 @@ package opportunities import ( "context" "fmt" - "reflect" - "strconv" "time" "github.com/MakeNowJust/heredoc/v2" @@ -16,72 +14,6 @@ import ( "github.com/spf13/viper" ) -type Opportunity struct { - ID string `json:"Id"` - Name string `json:"Name"` - Amount float64 `json:"Amount"` - StageName string `json:"StageName"` - CloseDate string `json:"CloseDate"` // Salesforce returns dates as strings - AccountID string `json:"AccountId"` - Probability float64 `json:"Probability"` - IsDeleted bool `json:"IsDeleted"` - Description string `json:"Description"` - Type string `json:"Type"` - NextStep string `json:"NextStep"` - LeadSource string `json:"LeadSource"` - IsClosed bool `json:"IsClosed"` - IsWon bool `json:"IsWon"` - ForecastCategory string `json:"ForecastCategory"` - ForecastCategoryName string `json:"ForecastCategoryName"` - CampaignID string `json:"CampaignId"` - HasOpportunityLineItem bool `json:"HasOpportunityLineItem"` - Pricebook2ID string `json:"Pricebook2Id"` - OwnerID string `json:"OwnerId"` - Territory2ID string `json:"Territory2Id"` - IsExcludedFromTerritory2Filter bool `json:"IsExcludedFromTerritory2Filter"` - CreatedDate string `json:"CreatedDate"` - CreatedById string `json:"CreatedById"` - LastModifiedDate string `json:"LastModifiedDate"` - LastModifiedById string `json:"LastModifiedById"` - SystemModstamp string `json:"SystemModstamp"` - LastActivityDate string `json:"LastActivityDate"` - PushCount int `json:"PushCount"` - LastStageChangeDate string `json:"LastStageChangeDate"` - ContactId string `json:"ContactId"` - LastViewedDate string `json:"LastViewedDate"` - LastReferencedDate string `json:"LastReferencedDate"` - SyncedQuoteId string `json:"SyncedQuoteId"` - ContractId string `json:"ContractId"` - HasOpenActivity bool `json:"HasOpenActivity"` - HasOverdueTask bool `json:"HasOverdueTask"` - LastAmountChangedHistoryId string `json:"LastAmountChangedHistoryId"` - LastCloseDateChangedHistoryId string `json:"LastCloseDateChangedHistoryId"` - - // CustomFields holds any additional fields not defined in the struct - CustomFields map[string]interface{} `json:"-"` -} - -// UnmarshalJSON implements custom unmarshalling to capture unknown fields -func (o *Opportunity) UnmarshalJSON(data []byte) error { - type Alias Opportunity - aux := &struct { - *Alias - }{ - Alias: (*Alias)(o), - } - - knownFields := common.GetKnownFieldsFromStruct(reflect.TypeOf(Opportunity{})) - - customFields, err := common.UnmarshalWithCustomFields(data, aux, knownFields) - if err != nil { - return err - } - - o.CustomFields = customFields - - return nil -} - func NewSalesforceOpportunitiesCmd() *cobra.Command { var name string var metadataMappings map[string]string @@ -163,13 +95,14 @@ func NewSalesforceOpportunitiesCmd() *cobra.Command { return cmd } +// processOpportunities queries and transforms opportunities func processOpportunities(ctx context.Context, sf *salesforce.Salesforce, metadataMappings map[string]string, limit int, listAllFields bool, whereClause string) ([]api.CreateResource, error) { additionalFields := make([]string, 0, len(metadataMappings)) for _, fieldName := range metadataMappings { additionalFields = append(additionalFields, fieldName) } - var opportunities []Opportunity + var opportunities []map[string]any err := common.QuerySalesforceObject(ctx, sf, "Opportunity", limit, listAllFields, &opportunities, additionalFields, whereClause) if err != nil { return nil, err @@ -186,77 +119,99 @@ func processOpportunities(ctx context.Context, sf *salesforce.Salesforce, metada return resources, nil } -func transformOpportunityToResource(opportunity Opportunity, metadataMappings map[string]string) api.CreateResource { - var closeDateFormatted string - if opportunity.CloseDate != "" { - if t, err := time.Parse("2006-01-02", opportunity.CloseDate); err == nil { - closeDateFormatted = t.Format(time.RFC3339) - } else { - closeDateFormatted = opportunity.CloseDate +func formatCloseDate(closeDate any) string { + if closeDate == nil { + return "" + } + + closeDateStr := fmt.Sprintf("%v", closeDate) + if str, ok := closeDate.(string); ok && str != "" { + if t, err := time.Parse("2006-01-02", str); err == nil { + return t.Format(time.RFC3339) } } + return closeDateStr +} - metadata := map[string]string{ - "opportunity/id": opportunity.ID, - "ctrlplane/external-id": opportunity.ID, - "opportunity/account-id": opportunity.AccountID, - "opportunity/stage": opportunity.StageName, - "opportunity/amount": strconv.FormatFloat(opportunity.Amount, 'f', -1, 64), - "opportunity/probability": strconv.FormatFloat(opportunity.Probability, 'f', -1, 64), - "opportunity/close-date": closeDateFormatted, - "opportunity/name": opportunity.Name, - "opportunity/type": opportunity.Type, - "opportunity/owner-id": opportunity.OwnerID, - "opportunity/is-closed": strconv.FormatBool(opportunity.IsClosed), - "opportunity/is-won": strconv.FormatBool(opportunity.IsWon), - "opportunity/lead-source": opportunity.LeadSource, - "opportunity/forecast-category": opportunity.ForecastCategory, - "opportunity/contact-id": opportunity.ContactId, - "opportunity/campaign-id": opportunity.CampaignID, - "opportunity/created-date": opportunity.CreatedDate, - "opportunity/last-modified": opportunity.LastModifiedDate, +func transformOpportunityToResource(opportunity map[string]any, metadataMappings map[string]string) api.CreateResource { + metadata := map[string]string{} + + common.AddToMetadata(metadata, "opportunity/id", opportunity["Id"]) + common.AddToMetadata(metadata, "opportunity/account-id", opportunity["AccountId"]) + common.AddToMetadata(metadata, "opportunity/stage", opportunity["StageName"]) + common.AddToMetadata(metadata, "opportunity/amount", opportunity["Amount"]) + common.AddToMetadata(metadata, "opportunity/probability", opportunity["Probability"]) + common.AddToMetadata(metadata, "opportunity/name", opportunity["Name"]) + common.AddToMetadata(metadata, "opportunity/type", opportunity["Type"]) + common.AddToMetadata(metadata, "opportunity/owner-id", opportunity["OwnerId"]) + common.AddToMetadata(metadata, "opportunity/is-closed", opportunity["IsClosed"]) + common.AddToMetadata(metadata, "opportunity/is-won", opportunity["IsWon"]) + common.AddToMetadata(metadata, "opportunity/lead-source", opportunity["LeadSource"]) + common.AddToMetadata(metadata, "opportunity/forecast-category", opportunity["ForecastCategory"]) + common.AddToMetadata(metadata, "opportunity/contact-id", opportunity["ContactId"]) + common.AddToMetadata(metadata, "opportunity/campaign-id", opportunity["CampaignId"]) + common.AddToMetadata(metadata, "opportunity/created-date", opportunity["CreatedDate"]) + common.AddToMetadata(metadata, "opportunity/last-modified", opportunity["LastModifiedDate"]) + + if closeDate := opportunity["CloseDate"]; closeDate != nil { + closeDateFormatted := fmt.Sprintf("%v", closeDate) + if closeDateStr, ok := closeDate.(string); ok && closeDateStr != "" { + if t, err := time.Parse("2006-01-02", closeDateStr); err == nil { + closeDateFormatted = t.Format(time.RFC3339) + } + } + metadata["opportunity/close-date"] = closeDateFormatted } for metadataKey, fieldName := range metadataMappings { - if value, found := common.GetCustomFieldValue(opportunity, fieldName); found { - metadata[metadataKey] = value + if value, exists := opportunity[fieldName]; exists { + common.AddToMetadata(metadata, metadataKey, value) + } + } + + fiscalQuarter := 0 + fiscalYear := 0 + if period, ok := opportunity["FiscalPeriod"].(string); ok && period != "" { + // Period format is typically like "2024 Q1" + if n, err := fmt.Sscanf(period, "%d Q%d", &fiscalYear, &fiscalQuarter); err != nil || n != 2 { + log.Debug("Failed to parse fiscal period", "period", period, "error", err) + // Leave fiscalQuarter and fiscalYear as 0 } } config := map[string]interface{}{ - "name": opportunity.Name, - "amount": strconv.FormatFloat(opportunity.Amount, 'f', -1, 64), - "stage": opportunity.StageName, - "id": opportunity.ID, + "name": fmt.Sprintf("%v", opportunity["Name"]), + "amount": opportunity["Amount"], + "stage": fmt.Sprintf("%v", opportunity["StageName"]), + "id": fmt.Sprintf("%v", opportunity["Id"]), + "probability": opportunity["Probability"], + "isClosed": opportunity["IsClosed"], + "isWon": opportunity["IsWon"], "salesforceOpportunity": map[string]interface{}{ - "recordId": opportunity.ID, - "closeDate": closeDateFormatted, - "accountId": opportunity.AccountID, - "probability": strconv.FormatFloat(opportunity.Probability, 'f', -1, 64), - "type": opportunity.Type, - "description": opportunity.Description, - "nextStep": opportunity.NextStep, - "leadSource": opportunity.LeadSource, - "isClosed": opportunity.IsClosed, - "isWon": opportunity.IsWon, - "forecastCategory": opportunity.ForecastCategory, - "ownerId": opportunity.OwnerID, - "contactId": opportunity.ContactId, - "campaignId": opportunity.CampaignID, - "hasLineItems": opportunity.HasOpportunityLineItem, - "createdDate": opportunity.CreatedDate, - "lastModifiedDate": opportunity.LastModifiedDate, - "pushCount": opportunity.PushCount, - "lastStageChangeDate": opportunity.LastStageChangeDate, + "recordId": fmt.Sprintf("%v", opportunity["Id"]), + "accountId": fmt.Sprintf("%v", opportunity["AccountId"]), + "ownerId": fmt.Sprintf("%v", opportunity["OwnerId"]), + "type": fmt.Sprintf("%v", opportunity["Type"]), + "leadSource": fmt.Sprintf("%v", opportunity["LeadSource"]), + "closeDate": formatCloseDate(opportunity["CloseDate"]), + "forecastCategory": fmt.Sprintf("%v", opportunity["ForecastCategory"]), + "description": fmt.Sprintf("%v", opportunity["Description"]), + "nextStep": fmt.Sprintf("%v", opportunity["NextStep"]), + "hasOpenActivity": opportunity["HasOpenActivity"], + "createdDate": fmt.Sprintf("%v", opportunity["CreatedDate"]), + "lastModifiedDate": fmt.Sprintf("%v", opportunity["LastModifiedDate"]), + "lastActivityDate": fmt.Sprintf("%v", opportunity["LastActivityDate"]), + "fiscalQuarter": fiscalQuarter, + "fiscalYear": fiscalYear, }, } return api.CreateResource{ Version: "ctrlplane.dev/crm/opportunity/v1", Kind: "SalesforceOpportunity", - Name: opportunity.Name, - Identifier: opportunity.ID, + Name: fmt.Sprintf("%v", opportunity["Name"]), + Identifier: fmt.Sprintf("%v", opportunity["Id"]), Config: config, Metadata: metadata, } diff --git a/cmd/ctrlc/root/sync/salesforce/salesforce.go b/cmd/ctrlc/root/sync/salesforce/salesforce.go index 7d3c635..e0e385a 100644 --- a/cmd/ctrlc/root/sync/salesforce/salesforce.go +++ b/cmd/ctrlc/root/sync/salesforce/salesforce.go @@ -1,6 +1,8 @@ package salesforce import ( + "fmt" + "github.com/MakeNowJust/heredoc" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/accounts" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/salesforce/opportunities" @@ -27,17 +29,31 @@ func NewSalesforceCmd() *cobra.Command { `), } - cmd.PersistentFlags().String("salesforce-domain", "", "Salesforce domain (e.g., https://my-domain.my.salesforce.com)") - cmd.PersistentFlags().String("salesforce-consumer-key", "", "Salesforce consumer key") - cmd.PersistentFlags().String("salesforce-consumer-secret", "", "Salesforce consumer secret") + cmd.PersistentFlags().String("salesforce-domain", "", "Salesforce domain (e.g., https://my-domain.my.salesforce.com) (can also be set via SALESFORCE_DOMAIN env var)") + cmd.PersistentFlags().String("salesforce-consumer-key", "", "Salesforce consumer key (can also be set via SALESFORCE_CONSUMER_KEY env var)") + cmd.PersistentFlags().String("salesforce-consumer-secret", "", "Salesforce consumer secret (can also be set via SALESFORCE_CONSUMER_SECRET env var)") + + viper.AutomaticEnv() - viper.BindPFlag("salesforce-domain", cmd.PersistentFlags().Lookup("salesforce-domain")) - viper.BindPFlag("salesforce-consumer-key", cmd.PersistentFlags().Lookup("salesforce-consumer-key")) - viper.BindPFlag("salesforce-consumer-secret", cmd.PersistentFlags().Lookup("salesforce-consumer-secret")) + if err := viper.BindEnv("salesforce-domain", "SALESFORCE_DOMAIN"); err != nil { + panic(fmt.Errorf("failed to bind SALESFORCE_DOMAIN env var: %w", err)) + } + if err := viper.BindEnv("salesforce-consumer-key", "SALESFORCE_CONSUMER_KEY"); err != nil { + panic(fmt.Errorf("failed to bind SALESFORCE_CONSUMER_KEY env var: %w", err)) + } + if err := viper.BindEnv("salesforce-consumer-secret", "SALESFORCE_CONSUMER_SECRET"); err != nil { + panic(fmt.Errorf("failed to bind SALESFORCE_CONSUMER_SECRET env var: %w", err)) + } - cmd.MarkPersistentFlagRequired("salesforce-domain") - cmd.MarkPersistentFlagRequired("salesforce-consumer-key") - cmd.MarkPersistentFlagRequired("salesforce-consumer-secret") + if err := viper.BindPFlag("salesforce-domain", cmd.PersistentFlags().Lookup("salesforce-domain")); err != nil { + panic(fmt.Errorf("failed to bind salesforce-domain flag: %w", err)) + } + if err := viper.BindPFlag("salesforce-consumer-key", cmd.PersistentFlags().Lookup("salesforce-consumer-key")); err != nil { + panic(fmt.Errorf("failed to bind salesforce-consumer-key flag: %w", err)) + } + if err := viper.BindPFlag("salesforce-consumer-secret", cmd.PersistentFlags().Lookup("salesforce-consumer-secret")); err != nil { + panic(fmt.Errorf("failed to bind salesforce-consumer-secret flag: %w", err)) + } cmd.AddCommand(accounts.NewSalesforceAccountsCmd()) cmd.AddCommand(opportunities.NewSalesforceOpportunitiesCmd()) From 5eed7cfe4ef537b466141b198881c864a02166fa Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk <77289967+zacharyblasczyk@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:23:49 -0700 Subject: [PATCH 6/8] Update cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../sync/salesforce/opportunities/opportunities.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go index 75dd0f5..3dcf419 100644 --- a/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go +++ b/cmd/ctrlc/root/sync/salesforce/opportunities/opportunities.go @@ -153,14 +153,8 @@ func transformOpportunityToResource(opportunity map[string]any, metadataMappings common.AddToMetadata(metadata, "opportunity/created-date", opportunity["CreatedDate"]) common.AddToMetadata(metadata, "opportunity/last-modified", opportunity["LastModifiedDate"]) - if closeDate := opportunity["CloseDate"]; closeDate != nil { - closeDateFormatted := fmt.Sprintf("%v", closeDate) - if closeDateStr, ok := closeDate.(string); ok && closeDateStr != "" { - if t, err := time.Parse("2006-01-02", closeDateStr); err == nil { - closeDateFormatted = t.Format(time.RFC3339) - } - } - metadata["opportunity/close-date"] = closeDateFormatted + if closeDate := formatCloseDate(opportunity["CloseDate"]); closeDate != "" { + metadata["opportunity/close-date"] = closeDate } for metadataKey, fieldName := range metadataMappings { From 3702dc8771b7d9726a709c2063477e8f664b9d98 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk <77289967+zacharyblasczyk@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:24:00 -0700 Subject: [PATCH 7/8] Update cmd/ctrlc/root/sync/salesforce/common/util.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- cmd/ctrlc/root/sync/salesforce/common/util.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/ctrlc/root/sync/salesforce/common/util.go b/cmd/ctrlc/root/sync/salesforce/common/util.go index 83a87ef..13e74dd 100644 --- a/cmd/ctrlc/root/sync/salesforce/common/util.go +++ b/cmd/ctrlc/root/sync/salesforce/common/util.go @@ -121,9 +121,19 @@ func buildSOQL(objectName string, fields []string, whereClause string, lastId st if whereClause != "" { conditions = append(conditions, whereClause) } - if lastId != "" { - conditions = append(conditions, fmt.Sprintf("Id > '%s'", lastId)) - } + if lastId != "" { + // Reject anything that isn’t exactly 15 or 18 alphanumeric chars + if len(lastId) != 15 && len(lastId) != 18 { + return "", fmt.Errorf("invalid Salesforce ID: %q", lastId) + } + for i := range lastId { + c := lastId[i] + if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) { + return "", fmt.Errorf("invalid Salesforce ID: %q", lastId) + } + } + conditions = append(conditions, fmt.Sprintf("Id > '%s'", lastId)) + } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") } From 3226b300944a4ed7a5f66fc7ae40c6555ab527ee Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Wed, 16 Jul 2025 14:28:56 -0700 Subject: [PATCH 8/8] fix --- cmd/ctrlc/root/sync/salesforce/common/util.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/cmd/ctrlc/root/sync/salesforce/common/util.go b/cmd/ctrlc/root/sync/salesforce/common/util.go index 13e74dd..83a87ef 100644 --- a/cmd/ctrlc/root/sync/salesforce/common/util.go +++ b/cmd/ctrlc/root/sync/salesforce/common/util.go @@ -121,19 +121,9 @@ func buildSOQL(objectName string, fields []string, whereClause string, lastId st if whereClause != "" { conditions = append(conditions, whereClause) } - if lastId != "" { - // Reject anything that isn’t exactly 15 or 18 alphanumeric chars - if len(lastId) != 15 && len(lastId) != 18 { - return "", fmt.Errorf("invalid Salesforce ID: %q", lastId) - } - for i := range lastId { - c := lastId[i] - if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) { - return "", fmt.Errorf("invalid Salesforce ID: %q", lastId) - } - } - conditions = append(conditions, fmt.Sprintf("Id > '%s'", lastId)) - } + if lastId != "" { + conditions = append(conditions, fmt.Sprintf("Id > '%s'", lastId)) + } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") }