Single device id with configurable default (#332)

* Single device id with configurable default
* Rename advancingCookie middleware to identity middleware

.VERSION

@@ -1 +1 @@
-v0.11.1
+v0.11.2

cmd/app.go

@@ -133,10 +133,6 @@ func (a *App) initializeMiddleware() {
 		limiterMiddleware := middleware.BuildRateLimiterMiddleware(limiter)
 		a.engine.Use(limiterMiddleware)
 	}
-	if a.config.Middleware.Cookie.Enabled {
-		log.Info().Msg("initializing advancing cookie middleware")
-		a.engine.Use(middleware.AdvancingCookie(a.config.Cookie))
-	}
 	if a.config.Middleware.Cors.Enabled {
 		log.Info().Msg("initializing cors middleware")
 		a.engine.Use(middleware.CORS(a.config.Middleware.Cors))
@@ -149,6 +145,8 @@ func (a *App) initializeMiddleware() {
 		log.Info().Msg("initializing yeet middleware")
 		a.engine.Use(middleware.Yeet())
 	}
+	log.Info().Msg("initializing identity middleware")
+	a.engine.Use(middleware.Identity(a.config.Identity))
 }
 
 func (a *App) initializeHealthcheckRoutes() {

examples/quickstart/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.9"
 services:
   honeypot:
     container_name: honeypot
-    image: ghcr.io/silverton-io/honeypot:v0.11.1
+    image: ghcr.io/silverton-io/honeypot:v0.11.2
     volumes:
       - type: bind
         source: ./honeypot/quickstart.conf.yml

examples/quickstart/honeypot/quickstart.conf.yml

@@ -20,14 +20,16 @@ middleware:
     enabled: false
     period: S
     limit: 10
-  cookie:
-    enabled: true
-    name: sp-nuid
-    secure: false
-    ttlDays: 365
-    domain: localhost
-    path: /
-    sameSite: Lax
+  identity:
+    cookie:
+      enabled: true
+      name: nuid
+      secure: true
+      ttlDays: 365
+      domain: ""
+      path: /
+      sameSite: Lax
+    fallback: 00000000-0000-4000-A000-000000000000
   cors:
     enabled: true
     allowOrigin:
@@ -53,10 +55,10 @@ inputs:
     redirectPath: /plw/r
   cloudevents:
     enabled: true
-    path: /ce/p
+    path: /cloudevents
   generic:
     enabled: true
-    path: /gen/p
+    path: /generic
     contexts:
       rootKey: contexts
     payload:
@@ -70,10 +72,6 @@ inputs:
     enabled: true
     path: /pxl
 
-  relay:
-    enabled: true
-    path: /relay
-
 schemaCache:
   backend:
     type: fs

pkg/config/middleware.go

@@ -3,7 +3,7 @@ package config
 type Middleware struct {
 	Timeout       `json:"timeout"`
 	RateLimiter   `json:"rateLimiter"`
-	Cookie        `json:"cookie"`
+	Identity      `json:"identity"`
 	Cors          `json:"cors"`
 	RequestLogger `json:"requestLogger"`
 	Yeet          `json:"yeet"`
@@ -20,7 +20,12 @@ type RateLimiter struct {
 	Limit   int64  `json:"limit"`
 }
 
-type Cookie struct {
+type Identity struct {
+	Cookie   IdentityCookie `json:"cookie"`
+	Fallback string         `json:"fallback"`
+}
+
+type IdentityCookie struct {
 	Enabled  bool   `json:"enabled"`
 	Name     string `json:"name"`
 	Secure   bool   `json:"secure"`

pkg/envelope/builderCloudevent.go

@@ -20,7 +20,7 @@ func BuildCloudeventEnvelopesFromRequest(c *gin.Context, conf *config.Config, m
 		return envelopes
 	}
 	for _, ce := range gjson.ParseBytes(reqBody).Array() {
-		n := buildCommonEnvelope(c, m)
+		n := buildCommonEnvelope(c, conf.Middleware, m)
 		cEvent, err := cloudevents.BuildEvent(ce)
 		if err != nil {
 			log.Error().Err(err).Msg("could not build Cloudevent")

pkg/envelope/builderCommon.go

@@ -5,12 +5,15 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
+	"github.com/silverton-io/honeypot/pkg/config"
 	"github.com/silverton-io/honeypot/pkg/constants"
 	"github.com/silverton-io/honeypot/pkg/event"
 	"github.com/silverton-io/honeypot/pkg/meta"
+	"github.com/silverton-io/honeypot/pkg/util"
 )
 
-func buildCommonEnvelope(c *gin.Context, m *meta.CollectorMeta) Envelope {
+func buildCommonEnvelope(c *gin.Context, conf config.Middleware, m *meta.CollectorMeta) Envelope {
+	identity := util.GetIdentityOrFallback(c, conf)
 	envelope := Envelope{
 		EventMeta: EventMeta{
 			Uuid:      uuid.New(),
@@ -32,6 +35,7 @@ func buildCommonEnvelope(c *gin.Context, m *meta.CollectorMeta) Envelope {
 		},
 		Device: Device{
 			Ip:        c.ClientIP(),
+			Id:        identity,
 			Useragent: c.Request.UserAgent(),
 		},
 		User:       User{},

pkg/envelope/builderGeneric.go

@@ -20,7 +20,7 @@ func BuildGenericEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *me
 		return envelopes
 	}
 	for _, e := range gjson.ParseBytes(reqBody).Array() {
-		n := buildCommonEnvelope(c, m)
+		n := buildCommonEnvelope(c, conf.Middleware, m)
 		genEvent, err := generic.BuildEvent(e, conf.Generic)
 		if err != nil {
 			log.Error().Err(err).Msg("could not build generic event")

pkg/envelope/builderPixel.go

@@ -4,7 +4,6 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/rs/zerolog/log"
 	"github.com/silverton-io/honeypot/pkg/config"
-	"github.com/silverton-io/honeypot/pkg/constants"
 	"github.com/silverton-io/honeypot/pkg/meta"
 	"github.com/silverton-io/honeypot/pkg/pixel"
 	"github.com/silverton-io/honeypot/pkg/protocol"
@@ -13,15 +12,12 @@ import (
 // NOTE - one envelope per request
 func BuildPixelEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *meta.CollectorMeta) []Envelope {
 	var envelopes []Envelope
-	nid := c.GetString(constants.IDENTITY)
 	sde, err := pixel.BuildEvent(c)
 	if err != nil {
 		log.Error().Err(err).Msg("could not build pixel event")
 	}
 	contexts := buildContextsFromRequest(c)
-	n := buildCommonEnvelope(c, m)
-	// Device
-	n.Device.Nid = nid
+	n := buildCommonEnvelope(c, conf.Middleware, m)
 	// Event Meta
 	n.EventMeta.Protocol = protocol.PIXEL
 	n.EventMeta.Schema = sde.Schema

pkg/envelope/builderSnowplow.go

@@ -6,26 +6,25 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/rs/zerolog/log"
 	"github.com/silverton-io/honeypot/pkg/config"
-	"github.com/silverton-io/honeypot/pkg/constants"
 	"github.com/silverton-io/honeypot/pkg/meta"
 	"github.com/silverton-io/honeypot/pkg/protocol"
 	"github.com/silverton-io/honeypot/pkg/snowplow"
 	"github.com/silverton-io/honeypot/pkg/util"
 	"github.com/tidwall/gjson"
 )
 
-func buildSnowplowEnvelope(c *gin.Context, e snowplow.SnowplowEvent, m *meta.CollectorMeta) Envelope {
-	nid := c.GetString(constants.IDENTITY)
-	n := buildCommonEnvelope(c, m)
+func buildSnowplowEnvelope(c *gin.Context, conf *config.Config, e snowplow.SnowplowEvent, m *meta.CollectorMeta) Envelope {
+	n := buildCommonEnvelope(c, conf.Middleware, m)
 	// Event Meta
 	n.EventMeta.Protocol = protocol.SNOWPLOW
 	n.EventMeta.Schema = *e.SelfDescribingEvent.SchemaName()
 	// Pipeline
 	n.Pipeline.Source.GeneratedTstamp = e.DvceCreatedTstamp
 	n.Pipeline.Source.SentTstamp = e.DvceSentTstamp
 	// Device
-	n.Device.Id = *e.DomainUserid
-	n.Device.Nid = nid
+	if e.DomainUserid != nil {
+		n.Device.Id = *e.DomainUserid
+	}
 	n.Device.Os = Os{Timezone: e.OsTimezone}
 	n.Device.Browser = Browser{
 		Lang:           e.BrLang,
@@ -98,13 +97,13 @@ func BuildSnowplowEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *m
 		payloadData := gjson.GetBytes(body, "data")
 		for _, event := range payloadData.Array() {
 			spEvent := snowplow.BuildEventFromMappedParams(c, event.Value().(map[string]interface{}), *conf)
-			e := buildSnowplowEnvelope(c, spEvent, m)
+			e := buildSnowplowEnvelope(c, conf, spEvent, m)
 			envelopes = append(envelopes, e)
 		}
 	} else {
 		params := util.MapUrlParams(c)
 		spEvent := snowplow.BuildEventFromMappedParams(c, params, *conf)
-		e := buildSnowplowEnvelope(c, spEvent, m)
+		e := buildSnowplowEnvelope(c, conf, spEvent, m)
 		envelopes = append(envelopes, e)
 	}
 	return envelopes

pkg/envelope/builderWebhook.go

@@ -20,7 +20,7 @@ func BuildWebhookEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *me
 		return envelopes
 	}
 	for _, e := range gjson.ParseBytes(reqBody).Array() {
-		n := buildCommonEnvelope(c, m)
+		n := buildCommonEnvelope(c, conf.Middleware, m)
 		contexts := buildContextsFromRequest(c)
 		sde, err := webhook.BuildEvent(c, e)
 		if err != nil {

pkg/envelope/device.go

@@ -8,8 +8,7 @@ import (
 type Device struct {
 	Ip                string  `json:"ip"`
 	Useragent         string  `json:"useragent"`
-	Id                string  `json:"id"`
-	Nid               string  `json:"nid"`
+	Id                string  `json:"id"`                  // Server-side cookie, header, or query params
 	Idfa              *string `json:"idfa,omitempty"`      // [iOS]
 	Idfv              *string `json:"idfv,omitempty"`      // [iOS]
 	AdId              *string `json:"adId,omitempty"`      // [Android] Google play services advertising id

pkg/middleware/advancingCookie.go → pkg/middleware/identity.go

@@ -9,10 +9,10 @@ import (
 	"github.com/silverton-io/honeypot/pkg/constants"
 )
 
-func AdvancingCookie(conf config.Cookie) gin.HandlerFunc {
+func Identity(conf config.Identity) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		identityCookieValue, _ := c.Cookie(conf.Name)
-		switch conf.SameSite {
+		identityCookieValue, _ := c.Cookie(conf.Cookie.Name)
+		switch conf.Cookie.SameSite {
 		case "None":
 			c.SetSameSite(http.SameSiteNoneMode)
 		case "Lax":
@@ -22,23 +22,23 @@ func AdvancingCookie(conf config.Cookie) gin.HandlerFunc {
 		}
 		if identityCookieValue != "" {
 			c.SetCookie(
-				conf.Name,
+				conf.Cookie.Name,
 				identityCookieValue,
-				60*60*24*conf.TtlDays,
-				conf.Path,
-				conf.Domain,
-				conf.Secure,
+				60*60*24*conf.Cookie.TtlDays,
+				conf.Cookie.Path,
+				conf.Cookie.Domain,
+				conf.Cookie.Secure,
 				false,
 			)
 		} else {
 			identityCookieValue = uuid.New().String()
 			c.SetCookie(
-				conf.Name,
+				conf.Cookie.Name,
 				identityCookieValue,
-				60*60*24*conf.TtlDays,
-				conf.Path,
-				conf.Domain,
-				conf.Secure,
+				60*60*24*conf.Cookie.TtlDays,
+				conf.Cookie.Path,
+				conf.Cookie.Domain,
+				conf.Cookie.Secure,
 				false,
 			)
 		}

pkg/sink/amplitude.go

@@ -120,7 +120,7 @@ func (s *AmplitudeSink) batchPublish(ctx context.Context, envelopes []envelope.E
 		}
 		evnt := amplitudeEvent{
 			Time:            e.Pipeline.Source.GeneratedTstamp.Unix(),
-			DeviceId:        e.Device.Nid,
+			DeviceId:        e.Device.Id,
 			EventType:       e.EventMeta.Namespace,
 			EventProperties: flattenedEnvelope,
 			InsertId:        e.EventMeta.Uuid.String(),

pkg/util/identity.go

@@ -0,0 +1,15 @@
+package util
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/silverton-io/honeypot/pkg/config"
+	"github.com/silverton-io/honeypot/pkg/constants"
+)
+
+func GetIdentityOrFallback(c *gin.Context, conf config.Middleware) string {
+	identity := c.GetString(constants.IDENTITY)
+	if identity == "" {
+		return conf.Fallback
+	}
+	return identity
+}