feat: generic consent management (#4056)

* feat: add support for generic consent management

* fix: types and consent management data in utilities

* refactor: use types

* fix: syntax issues

* fix: use make

* fix: all syntax issues

* fix: formatting

* test: use correct data type

* chore: fix formatting

* fix: field types

* test: correct test data

* fix: field extraction

* test: add additional test case

* test: add more scenarios for legacy consent management

* chore: fix formatting

* fix: conditional check

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>

* fix: conditional check

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>

* refactor: use slices utility method

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>

* fix: handle undefined destination id

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>

* refactor: move consent management code to a separate file

* chore: empty commit

* fix: consent config resolution

* test: add test cases for generic consent management

* test: fix assertion values in test cases

* test: fix expected value

* refactor: remove unnecessary type

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>

* refactor: add inline logic

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>

* refactor: remove unwanted function

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>

* test: add unit tests for consentmanagementfilter

* chore: format files

* test: add more unit tests for consentmanagementfilter

* chore: merge latest changes from master

* refactor: delete consentmanagementfilter package

* fix: test script

* chore: empty commit

* fix: test script

* refactor: update gcm logic and test suite

* refactor: update method names

* test: shuffle tests

* test: fix expected value

* test: fix expected value

* test: fix test suites

* test: add more test cases for coverage

* test: fix input and expected data

* test: fix input and expected data

* chore: skip a test case

* test: fix issues with test script

* chore: fix formatting

* refactor: rename variables and add documentation

* refactor: add logger statements

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* chore: add logger statements for debugging

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* fix: an edge case in GetKetchConsentCategories

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* Update processor/consent.go

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* fix: syntax issue

* fix: syntax issue

* fix: logger statements

* chore: fix lint issue

* fix: do not expose methods unnecessarily

* fix: idomatic capitalization

* chore: add comment to clarify

* Update processor/consent.go

Co-authored-by: Akash Chetty <achetty.iitr@gmail.com>

* fix: idomatic capitalization

* refactor: return early

* fix: ignore allowedConsentIds

---------

Co-authored-by: Rohith BCS <rohith.bcs@gmail.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Akash Chetty <achetty.iitr@gmail.com>

processor/consent.go

@@ -1,59 +1,132 @@
 package processor
 
 import (
+	"fmt"
+
 	"github.com/samber/lo"
 
 	backendconfig "github.com/rudderlabs/rudder-server/backend-config"
 	"github.com/rudderlabs/rudder-server/utils/misc"
 	"github.com/rudderlabs/rudder-server/utils/types"
 )
 
-// filterDestinations filters destinations based on consent categories, supports oneTrustCookieCategories and ketchConsentPurposes
-func (proc *Handle) filterDestinations(event types.SingularEventT, destinations []backendconfig.DestinationT) []backendconfig.DestinationT {
-	// If there are no deniedConsentIds, then return all destinations
-	deniedCategories := deniedConsentCategories(event)
-	if len(deniedCategories) == 0 {
+type ConsentManagementInfo struct {
+	DeniedConsentIDs   []string    `json:"deniedConsentIds"`
+	AllowedConsentIDs  interface{} `json:"allowedConsentIds"` // Not used currently but added for future use
+	Provider           string      `json:"provider"`
+	ResolutionStrategy string      `json:"resolutionStrategy"`
+}
+
+type GenericConsentManagementProviderData struct {
+	ResolutionStrategy string
+	Consents           []string
+}
+
+type GenericConsentsConfig struct {
+	Consent string `json:"consent"`
+}
+
+type GenericConsentManagementProviderConfig struct {
+	Provider           string                  `json:"provider"`
+	ResolutionStrategy string                  `json:"resolutionStrategy"`
+	Consents           []GenericConsentsConfig `json:"consents"`
+}
+
+/*
+Filters and returns destinations based on the consents configured for the destination and the user consents present in the event.
+
+Supports legacy and generic consent management.
+*/
+func (proc *Handle) getConsentFilteredDestinations(event types.SingularEventT, destinations []backendconfig.DestinationT) []backendconfig.DestinationT {
+	// If the event does not have denied consent IDs, do not filter any destinations
+	consentManagementInfo, err := getConsentManagementInfo(event)
+	if err != nil {
+		// Log the error for debugging purposes
+		proc.logger.Errorw("failed to get consent management info", "error", err.Error())
+	}
+
+	if len(consentManagementInfo.DeniedConsentIDs) == 0 {
 		return destinations
 	}
 
 	return lo.Filter(destinations, func(dest backendconfig.DestinationT, _ int) bool {
+		// This field differentiates legacy and generic consent management
+		if consentManagementInfo.Provider != "" {
+			// Generic consent management
+
+			if cmpData := proc.getGCMData(dest.ID, consentManagementInfo.Provider); len(cmpData.Consents) > 0 {
+
+				finalResolutionStrategy := consentManagementInfo.ResolutionStrategy
+
+				// For custom provider, the resolution strategy is to be picked from the destination config
+				if consentManagementInfo.Provider == "custom" {
+					finalResolutionStrategy = cmpData.ResolutionStrategy
+				}
+
+				switch finalResolutionStrategy {
+				// The user must consent to at least one of the configured consents in the destination
+				case "or":
+					return !lo.Every(consentManagementInfo.DeniedConsentIDs, cmpData.Consents)
+
+				// The user must consent to all of the configured consents in the destination
+				default: // "and"
+					return len(lo.Intersect(cmpData.Consents, consentManagementInfo.DeniedConsentIDs)) == 0
+				}
+			}
+			return true
+		}
+
+		// Legacy consent management
+
 		// If the destination has oneTrustCookieCategories, returns false if any of the oneTrustCategories are present in deniedCategories
-		if oneTrustCategories := proc.oneConsentCategories(dest.ID); len(oneTrustCategories) > 0 {
-			return len(lo.Intersect(oneTrustCategories, deniedCategories)) == 0
+		if oneTrustCategories := proc.getOneTrustConsentData(dest.ID); len(oneTrustCategories) > 0 {
+			return len(lo.Intersect(oneTrustCategories, consentManagementInfo.DeniedConsentIDs)) == 0
 		}
 
-		// If the destination has ketchConsentPurposes, returns false if all ketchCategories are present in deniedCategories
-		if ketchCategories := proc.ketchConsentCategories(dest.ID); len(ketchCategories) > 0 {
-			return !lo.Every(deniedCategories, ketchCategories)
+		// If the destination has ketchConsentPurposes, returns false if all ketchPurposes are present in deniedCategories
+		if ketchPurposes := proc.getKetchConsentData(dest.ID); len(ketchPurposes) > 0 {
+			return !lo.Every(consentManagementInfo.DeniedConsentIDs, ketchPurposes)
 		}
 		return true
 	})
 }
 
-func (proc *Handle) oneConsentCategories(destinationID string) []string {
+func (proc *Handle) getOneTrustConsentData(destinationID string) []string {
 	proc.config.configSubscriberLock.RLock()
 	defer proc.config.configSubscriberLock.RUnlock()
 	return proc.config.oneTrustConsentCategoriesMap[destinationID]
 }
 
-func (proc *Handle) ketchConsentCategories(destinationID string) []string {
+func (proc *Handle) getKetchConsentData(destinationID string) []string {
 	proc.config.configSubscriberLock.RLock()
 	defer proc.config.configSubscriberLock.RUnlock()
 	return proc.config.ketchConsentCategoriesMap[destinationID]
 }
 
-func deniedConsentCategories(se types.SingularEventT) []string {
-	if deniedConsents, _ := misc.MapLookup(se, "context", "consentManagement", "deniedConsentIds").([]interface{}); len(deniedConsents) > 0 {
-		return lo.FilterMap(deniedConsents, func(consent interface{}, _ int) (string, bool) {
-			consentStr, ok := consent.(string)
-			return consentStr, ok && consentStr != ""
-		})
+func (proc *Handle) getGCMData(destinationID, provider string) GenericConsentManagementProviderData {
+	proc.config.configSubscriberLock.RLock()
+	defer proc.config.configSubscriberLock.RUnlock()
+
+	defRetVal := GenericConsentManagementProviderData{}
+	destinationData, ok := proc.config.destGenericConsentManagementMap[destinationID]
+	if !ok {
+		return defRetVal
+	}
+
+	providerData, ok := destinationData[provider]
+	if !ok {
+		return defRetVal
 	}
-	return nil
+
+	return providerData
 }
 
-func oneTrustConsentCategories(dest *backendconfig.DestinationT) []string {
-	cookieCategories, _ := misc.MapLookup(dest.Config, "oneTrustCookieCategories").([]interface{})
+func getOneTrustConsentCategories(dest *backendconfig.DestinationT) []string {
+	cookieCategories, ok := misc.MapLookup(dest.Config, "oneTrustCookieCategories").([]interface{})
+	if !ok {
+		// Handle the case where oneTrustCookieCategories is not a slice
+		return nil
+	}
 	if len(cookieCategories) == 0 {
 		return nil
 	}
@@ -68,8 +141,12 @@ func oneTrustConsentCategories(dest *backendconfig.DestinationT) []string {
 	})
 }
 
-func ketchConsentCategories(dest *backendconfig.DestinationT) []string {
-	consentPurposes, _ := misc.MapLookup(dest.Config, "ketchConsentPurposes").([]interface{})
+func getKetchConsentCategories(dest *backendconfig.DestinationT) []string {
+	consentPurposes, ok := misc.MapLookup(dest.Config, "ketchConsentPurposes").([]interface{})
+	if !ok {
+		// Handle the case where ketchConsentPurposes is not a slice
+		return nil
+	}
 	if len(consentPurposes) == 0 {
 		return nil
 	}
@@ -83,3 +160,68 @@ func ketchConsentCategories(dest *backendconfig.DestinationT) []string {
 		}
 	})
 }
+
+func getGenericConsentManagementData(dest *backendconfig.DestinationT) (map[string]GenericConsentManagementProviderData, error) {
+	genericConsentManagementData := make(map[string]GenericConsentManagementProviderData)
+
+	if _, ok := dest.Config["consentManagement"]; !ok {
+		return genericConsentManagementData, nil
+	}
+
+	consentManagementConfigBytes, mErr := jsonfast.Marshal(dest.Config["consentManagement"])
+	if mErr != nil {
+		return genericConsentManagementData, fmt.Errorf("error marshalling consentManagement: %v for destination ID: %s", mErr, dest.ID)
+	}
+
+	consentManagementConfig := make([]GenericConsentManagementProviderConfig, 0)
+	unmErr := jsonfast.Unmarshal(consentManagementConfigBytes, &consentManagementConfig)
+
+	if unmErr != nil {
+		return genericConsentManagementData, fmt.Errorf("error unmarshalling consentManagementConfig: %v for destination ID: %s", unmErr, dest.ID)
+	}
+
+	for _, providerConfig := range consentManagementConfig {
+		consentsConfig := providerConfig.Consents
+
+		if len(consentsConfig) > 0 && providerConfig.Provider != "" {
+			consentIDs := lo.FilterMap(
+				consentsConfig,
+				func(consentsObj GenericConsentsConfig, _ int) (string, bool) {
+					return consentsObj.Consent, consentsObj.Consent != ""
+				},
+			)
+
+			if len(consentIDs) > 0 {
+				genericConsentManagementData[providerConfig.Provider] = GenericConsentManagementProviderData{
+					ResolutionStrategy: providerConfig.ResolutionStrategy,
+					Consents:           consentIDs,
+				}
+			}
+		}
+	}
+
+	return genericConsentManagementData, nil
+}
+
+func getConsentManagementInfo(event types.SingularEventT) (ConsentManagementInfo, error) {
+	consentManagementInfo := ConsentManagementInfo{}
+	if consentManagement, ok := misc.MapLookup(event, "context", "consentManagement").(map[string]interface{}); ok {
+		consentManagementObjBytes, mErr := jsonfast.Marshal(consentManagement)
+		if mErr != nil {
+			return consentManagementInfo, fmt.Errorf("error marshalling consentManagement: %v", mErr)
+		}
+
+		unmErr := jsonfast.Unmarshal(consentManagementObjBytes, &consentManagementInfo)
+		if unmErr != nil {
+			return consentManagementInfo, fmt.Errorf("error unmarshalling consentManagementInfo: %v", unmErr)
+		}
+
+		filterPredicate := func(consent string, _ int) (string, bool) {
+			return consent, consent != ""
+		}
+
+		consentManagementInfo.DeniedConsentIDs = lo.FilterMap(consentManagementInfo.DeniedConsentIDs, filterPredicate)
+	}
+
+	return consentManagementInfo, nil
+}