From 0ed2cc6d73d93866f9a5682be8a52c02c88b2cfe Mon Sep 17 00:00:00 2001 From: Matjaz Pirnovar Date: Thu, 16 Oct 2025 13:52:14 -0700 Subject: [PATCH 1/4] Add CMAB testing setup - Update go-sdk to v2.1.1 (latest master) for CMAB UUID support - Configure datafileURLTemplate for staging environment - Add CMAB testing documentation with test scenarios - Setup for testing with SDK key DCx4eoV52jhgaC9MSab3g --- config.yaml | 4 +- docs/cmab-testing.md | 500 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 4 files changed, 506 insertions(+), 4 deletions(-) create mode 100644 docs/cmab-testing.md diff --git a/config.yaml b/config.yaml index 283d2890..c9006e23 100644 --- a/config.yaml +++ b/config.yaml @@ -171,7 +171,9 @@ client: flushInterval: 30s ## Template URL for SDK datafile location. The template should specify a "%s" token for SDK key substitution. ## For secure environments, the datafileURLTemplate should be set to "https://config.optimizely.com/datafiles/auth/%s.json" - datafileURLTemplate: "https://cdn.optimizely.com/datafiles/%s.json" + # datafileURLTemplate: "https://cdn.optimizely.com/datafiles/%s.json" + # datafileURLTemplate: "https://dev.cdn.optimizely.com/datafiles/%s.json" + datafileURLTemplate: "https://optimizely-staging.s3.amazonaws.com/datafiles/%s.json" ## URL for dispatching events. eventURL: "https://logx.optimizely.com/v1/events" ## Validation Regex on the request SDK Key diff --git a/docs/cmab-testing.md b/docs/cmab-testing.md new file mode 100644 index 00000000..5ebb0c2e --- /dev/null +++ b/docs/cmab-testing.md @@ -0,0 +1,500 @@ +# CMAB Testing Guide for Agent + +## Overview + +This guide covers testing CMAB (Contextual Multi-Armed Bandit) functionality in Optimizely Agent. CMAB uses machine learning to dynamically optimize which variation a user sees based on their context. + +Agent exposes CMAB through the `/v1/decide` API. When a flag uses CMAB, the response includes a `cmabUUID` field that identifies the CMAB decision. + +## Setup + +### Prerequisites + +- Agent binary built and ready +- Optimizely project with a CMAB-enabled flag +- SDK key from your project +- curl command-line tool + +### Configure Agent + +Edit `config.yaml` to configure CMAB settings: + +```yaml +cmab: + requestTimeout: 10s + cache: + type: "memory" + size: 1000 + ttl: 30m + retryConfig: + maxRetries: 3 + initialBackoff: 100ms + maxBackoff: 10s + backoffMultiplier: 2.0 +``` + +### Set Environment Variables + +```bash +export OPTIMIZELY_SDK_KEY="your_sdk_key_here" +export FLAG_KEY="your_cmab_flag_key" +``` + +### Start Agent + +```bash +cd /Users/matjaz/repositories/agent +./bin/optimizely +``` + +Agent will start on http://localhost:8080 + +## Test Cases + +### Test 1: Basic CMAB Decision + +Make a decide request and verify it returns a cmabUUID. + +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "test_user_1", + "userAttributes": { + "age": 25, + "location": "San Francisco" + } + }' +``` + +Expected response should include: + +```json +{ + "flagKey": "your_flag_key", + "enabled": true, + "variationKey": "treatment", + "ruleKey": "cmab_rule", + "cmabUUID": "550e8400-e29b-41d4-a716-446655440000", + "variables": { ... } +} +``` + +What to verify: +- HTTP status is 200 +- Response contains cmabUUID field +- UUID is in valid format +- variationKey is returned + +### Test 2: Cache Behavior - Same Request Twice + +Make the same request twice and verify the second one uses cached results. + +First request: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_cache_test", + "userAttributes": {"age": 30} + }' +``` + +Copy the cmabUUID from the response, then make the exact same request again after waiting 1-2 seconds. + +What to verify: +- Both requests return the same cmabUUID +- Second request is faster (because it's cached) + +If the UUIDs are different, caching isn't working. + +### Test 3: Cache Miss When Attributes Change + +Verify that changing user attributes triggers a new CMAB decision. + +Request with age=25: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_attr_test", + "userAttributes": {"age": 25, "city": "NYC"} + }' +``` + +Note the cmabUUID. Then request with age=35: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_attr_test", + "userAttributes": {"age": 35, "city": "NYC"} + }' +``` + +What to verify: +- UUIDs are different (cache was invalidated) +- Both requests succeed +- Variations might be different based on CMAB model + +### Test 4: Different Users Get Different Cache Entries + +Request for user1: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_1", + "userAttributes": {"age": 30} + }' +``` + +Request for user2 with same attributes: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_2", + "userAttributes": {"age": 30} + }' +``` + +What to verify: +- Different cmabUUID for each user +- Both requests succeed + +### Test 5: Multiple Flags + +If you have multiple CMAB-enabled flags, test requesting them together: + +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=flag1&keys=flag2" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_multi", + "userAttributes": {"age": 28} + }' +``` + +Expected response is an array: +```json +[ + { + "flagKey": "flag1", + "cmabUUID": "uuid-1", + ... + }, + { + "flagKey": "flag2", + "cmabUUID": "uuid-2", + ... + } +] +``` + +What to verify: +- Array response with multiple decisions +- Each CMAB flag has its own cmabUUID +- Non-CMAB flags won't have a UUID + +### Test 6: Decide Options Work with CMAB + +Request with decide options: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_options", + "userAttributes": {"age": 32}, + "decideOptions": ["INCLUDE_REASONS", "EXCLUDE_VARIABLES"] + }' +``` + +What to verify: +- reasons array is populated +- variables field is excluded or empty +- cmabUUID is still present + +### Test 7: Cache TTL Expiration + +To test this, you need to temporarily change the TTL in config.yaml: + +```yaml +cmab: + cache: + ttl: 1m # Short TTL for testing +``` + +Restart Agent, then: +1. Make a request and note the cmabUUID +2. Wait 65 seconds +3. Make the exact same request again + +What to verify: +- Second request returns a different cmabUUID (cache expired) + +### Test 8: Reset Endpoint + +The reset endpoint clears the client cache. This is useful for test isolation. + +Make a decision request: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_reset", + "userAttributes": {"age": 40} + }' +``` + +Note the cmabUUID, then call reset: +```bash +curl -X POST "http://localhost:8080/v1/reset" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" +``` + +Should return: +```json +{"result": true} +``` + +Make the same decision request again. + +What to verify: +- Reset returns {"result": true} +- After reset, the same request may get a different cmabUUID + +### Test 9: Forced Decisions Bypass CMAB + +Request with a forced decision: +```bash +curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ + -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "user_forced", + "userAttributes": {"age": 50}, + "forcedDecisions": [ + { + "flagKey": "'${FLAG_KEY}'", + "variationKey": "control" + } + ] + }' +``` + +What to verify: +- Returns the forced variationKey ("control") +- CMAB is bypassed (may not have cmabUUID) + +### Test 10: Configuration Changes + +Test that config changes take effect: + +**Test different timeout:** +Edit config.yaml: +```yaml +cmab: + requestTimeout: 30s +``` +Restart Agent and verify requests succeed with longer timeout. + +**Test smaller cache:** +Edit config.yaml: +```yaml +cmab: + cache: + size: 10 +``` +Restart Agent, make 15 different requests (different users), then repeat request #1. You might get a different UUID because the cache was full. + +**Test retry config:** +Edit config.yaml: +```yaml +cmab: + retryConfig: + maxRetries: 5 +``` +Restart Agent and check logs for retry behavior when the prediction endpoint is slow. + +## Configuration Reference + +Here's what you can configure in config.yaml: + +```yaml +cmab: + # Timeout for CMAB prediction API calls + requestTimeout: 10s + + cache: + # Cache type: "memory" or "redis" + type: "memory" + + # Maximum cached entries (memory cache only) + size: 1000 + + # How long cache entries live + ttl: 30m + + retryConfig: + # How many times to retry failed requests + maxRetries: 3 + + # Starting backoff duration + initialBackoff: 100ms + + # Maximum backoff duration + maxBackoff: 10s + + # Backoff multiplier (exponential) + backoffMultiplier: 2.0 +``` + +Settings you might want to test: + +| Setting | Default | Try Testing | What Changes | +|---------|---------|-------------|--------------| +| requestTimeout | 10s | 5s, 30s | Request timeout behavior | +| cache.size | 1000 | 10, 100 | Cache eviction | +| cache.ttl | 30m | 1m, 60m | Cache expiration | +| maxRetries | 3 | 0, 5 | Retry attempts | + +## What Good Results Look Like + +When CMAB is working correctly: + +**Decide responses contain:** +- cmabUUID field with a valid UUID +- variationKey from the CMAB decision +- ruleKey referencing the CMAB rule + +**Caching works:** +- Same requests return the same UUID +- Changing attributes gets a new decision +- Different users get separate cache entries + +**Configuration is respected:** +- TTL expires cache at the right time +- Timeout prevents hanging +- Retries happen as configured + +**Endpoints respond:** +- /v1/decide returns decisions +- /v1/reset clears the cache + +## Common Issues + +### No cmabUUID in the response + +Possible reasons: +- Flag isn't configured for CMAB in your project +- User doesn't qualify for the CMAB rule (check attributes) +- Prediction endpoint isn't accessible + +Debug by checking Agent logs: +```bash +tail -f agent.log | grep -i cmab +``` + +### Cache not working + +If every request returns a different UUID: +- Check if TTL is too short +- Verify you're not changing userID or attributes +- Check if cache size is too small + +Enable debug logging: +```bash +# In config.yaml +log: + level: debug + +# Restart and check logs +./bin/optimizely 2>&1 | grep cache +``` + +### Requests timeout + +Check: +- Is the prediction endpoint responsive? +- Is requestTimeout too short? + +Increase timeout temporarily: +```yaml +cmab: + requestTimeout: 30s +``` + +## Test Results Template + +Document your test results: + +``` +CMAB Testing Results +Date: __________ +Tester: __________ +Agent Version: __________ +SDK Key: __________ + +Test 1: Basic CMAB Decision [ ] Pass [ ] Fail +Test 2: Cache - Same Request [ ] Pass [ ] Fail +Test 3: Cache - Attribute Change [ ] Pass [ ] Fail +Test 4: Cache - Different Users [ ] Pass [ ] Fail +Test 5: Multiple Flags [ ] Pass [ ] Fail [ ] N/A +Test 6: Decide Options [ ] Pass [ ] Fail +Test 7: Cache TTL Expiration [ ] Pass [ ] Fail +Test 8: Reset Endpoint [ ] Pass [ ] Fail +Test 9: Forced Decisions [ ] Pass [ ] Fail +Test 10: Configuration Changes [ ] Pass [ ] Fail + +Overall: [ ] Pass [ ] Fail + +Notes: +_______________________________________ +``` + +## Quick Reference + +**API Endpoints:** +- POST /v1/decide - Get feature decisions (includes CMAB) +- POST /v1/reset - Clear client cache +- GET /health - Health check + +**Required Headers:** +``` +X-Optimizely-SDK-Key: your_sdk_key +Content-Type: application/json +``` + +**Sample Request:** +```json +{ + "userId": "test_user", + "userAttributes": { + "age": 25, + "location": "SF" + } +} +``` + +**CMAB Response Fields:** +```json +{ + "flagKey": "string", + "enabled": true, + "variationKey": "string", + "ruleKey": "string", + "cmabUUID": "550e8400-e29b-41d4-a716-446655440000", + "variables": {}, + "userContext": {} +} +``` diff --git a/go.mod b/go.mod index 23eee746..ed98e3e4 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.3.1 github.com/lestrrat-go/jwx/v2 v2.0.20 - github.com/optimizely/go-sdk/v2 v2.0.0-20250820180618-907917e11924 + github.com/optimizely/go-sdk/v2 v2.1.1-0.20250930190916-92b83d299b7a github.com/orcaman/concurrent-map v1.0.0 github.com/prometheus/client_golang v1.18.0 github.com/rakyll/statik v0.1.7 diff --git a/go.sum b/go.sum index 28471778..6fd3b18e 100644 --- a/go.sum +++ b/go.sum @@ -229,8 +229,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/optimizely/go-sdk/v2 v2.0.0-20250820180618-907917e11924 h1:RxRZkvwvqMVEGphhGvs9zHT642g10ql+IDEDK7dcwZ4= -github.com/optimizely/go-sdk/v2 v2.0.0-20250820180618-907917e11924/go.mod h1:MusRCFsU7+XzJCoCTgheLoENJSf1iiFYm4KbJqz6BYA= +github.com/optimizely/go-sdk/v2 v2.1.1-0.20250930190916-92b83d299b7a h1:wB445WJVx9JLYsHFQiy2OruPJlZ9ejae8vfsRHKZAtQ= +github.com/optimizely/go-sdk/v2 v2.1.1-0.20250930190916-92b83d299b7a/go.mod h1:MusRCFsU7+XzJCoCTgheLoENJSf1iiFYm4KbJqz6BYA= github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= From 5ed62e501b72787c7fe65802c3d13bc10941b323 Mon Sep 17 00:00:00 2001 From: Matjaz Pirnovar Date: Thu, 16 Oct 2025 14:27:22 -0700 Subject: [PATCH 2/4] Update to go-sdk v2.1.1 client.CmabConfig API - Replace cmab.Config with client.CmabConfig - Remove RetryConfig parsing (now handled internally by go-sdk) - Simplify CMAB configuration to use stable public API --- pkg/optimizely/cache.go | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 31804375..64f3809f 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -333,34 +333,11 @@ func defaultLoader( cacheTTL = cmab.DefaultCacheTTL } - // Create retry config - retryConfig := &cmab.RetryConfig{ - MaxRetries: clientConf.CMAB.RetryConfig.MaxRetries, - InitialBackoff: clientConf.CMAB.RetryConfig.InitialBackoff, - MaxBackoff: clientConf.CMAB.RetryConfig.MaxBackoff, - BackoffMultiplier: clientConf.CMAB.RetryConfig.BackoffMultiplier, - } - - // Apply defaults for retry config if not set - if retryConfig.MaxRetries == 0 { - retryConfig.MaxRetries = cmab.DefaultMaxRetries - } - if retryConfig.InitialBackoff == 0 { - retryConfig.InitialBackoff = cmab.DefaultInitialBackoff - } - if retryConfig.MaxBackoff == 0 { - retryConfig.MaxBackoff = cmab.DefaultMaxBackoff - } - if retryConfig.BackoffMultiplier == 0 { - retryConfig.BackoffMultiplier = cmab.DefaultBackoffMultiplier - } - - // Create CMAB config (NO endpoint configuration - not configurable) - cmabConfig := cmab.Config{ + // Create CMAB config using client API (RetryConfig now handled internally by go-sdk) + cmabConfig := client.CmabConfig{ CacheSize: cacheSize, CacheTTL: cacheTTL, HTTPTimeout: clientConf.CMAB.RequestTimeout, - RetryConfig: retryConfig, } // Add to client options From 5e7b18bc0ac15baa4852312aac5eddaff4d056e2 Mon Sep 17 00:00:00 2001 From: Matjaz Pirnovar Date: Thu, 16 Oct 2025 14:40:22 -0700 Subject: [PATCH 3/4] Remove testing config changes - keep only go-sdk API update --- config.yaml | 4 +- docs/cmab-testing.md | 500 ------------------------------------------- 2 files changed, 1 insertion(+), 503 deletions(-) delete mode 100644 docs/cmab-testing.md diff --git a/config.yaml b/config.yaml index c9006e23..283d2890 100644 --- a/config.yaml +++ b/config.yaml @@ -171,9 +171,7 @@ client: flushInterval: 30s ## Template URL for SDK datafile location. The template should specify a "%s" token for SDK key substitution. ## For secure environments, the datafileURLTemplate should be set to "https://config.optimizely.com/datafiles/auth/%s.json" - # datafileURLTemplate: "https://cdn.optimizely.com/datafiles/%s.json" - # datafileURLTemplate: "https://dev.cdn.optimizely.com/datafiles/%s.json" - datafileURLTemplate: "https://optimizely-staging.s3.amazonaws.com/datafiles/%s.json" + datafileURLTemplate: "https://cdn.optimizely.com/datafiles/%s.json" ## URL for dispatching events. eventURL: "https://logx.optimizely.com/v1/events" ## Validation Regex on the request SDK Key diff --git a/docs/cmab-testing.md b/docs/cmab-testing.md deleted file mode 100644 index 5ebb0c2e..00000000 --- a/docs/cmab-testing.md +++ /dev/null @@ -1,500 +0,0 @@ -# CMAB Testing Guide for Agent - -## Overview - -This guide covers testing CMAB (Contextual Multi-Armed Bandit) functionality in Optimizely Agent. CMAB uses machine learning to dynamically optimize which variation a user sees based on their context. - -Agent exposes CMAB through the `/v1/decide` API. When a flag uses CMAB, the response includes a `cmabUUID` field that identifies the CMAB decision. - -## Setup - -### Prerequisites - -- Agent binary built and ready -- Optimizely project with a CMAB-enabled flag -- SDK key from your project -- curl command-line tool - -### Configure Agent - -Edit `config.yaml` to configure CMAB settings: - -```yaml -cmab: - requestTimeout: 10s - cache: - type: "memory" - size: 1000 - ttl: 30m - retryConfig: - maxRetries: 3 - initialBackoff: 100ms - maxBackoff: 10s - backoffMultiplier: 2.0 -``` - -### Set Environment Variables - -```bash -export OPTIMIZELY_SDK_KEY="your_sdk_key_here" -export FLAG_KEY="your_cmab_flag_key" -``` - -### Start Agent - -```bash -cd /Users/matjaz/repositories/agent -./bin/optimizely -``` - -Agent will start on http://localhost:8080 - -## Test Cases - -### Test 1: Basic CMAB Decision - -Make a decide request and verify it returns a cmabUUID. - -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "test_user_1", - "userAttributes": { - "age": 25, - "location": "San Francisco" - } - }' -``` - -Expected response should include: - -```json -{ - "flagKey": "your_flag_key", - "enabled": true, - "variationKey": "treatment", - "ruleKey": "cmab_rule", - "cmabUUID": "550e8400-e29b-41d4-a716-446655440000", - "variables": { ... } -} -``` - -What to verify: -- HTTP status is 200 -- Response contains cmabUUID field -- UUID is in valid format -- variationKey is returned - -### Test 2: Cache Behavior - Same Request Twice - -Make the same request twice and verify the second one uses cached results. - -First request: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_cache_test", - "userAttributes": {"age": 30} - }' -``` - -Copy the cmabUUID from the response, then make the exact same request again after waiting 1-2 seconds. - -What to verify: -- Both requests return the same cmabUUID -- Second request is faster (because it's cached) - -If the UUIDs are different, caching isn't working. - -### Test 3: Cache Miss When Attributes Change - -Verify that changing user attributes triggers a new CMAB decision. - -Request with age=25: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_attr_test", - "userAttributes": {"age": 25, "city": "NYC"} - }' -``` - -Note the cmabUUID. Then request with age=35: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_attr_test", - "userAttributes": {"age": 35, "city": "NYC"} - }' -``` - -What to verify: -- UUIDs are different (cache was invalidated) -- Both requests succeed -- Variations might be different based on CMAB model - -### Test 4: Different Users Get Different Cache Entries - -Request for user1: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_1", - "userAttributes": {"age": 30} - }' -``` - -Request for user2 with same attributes: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_2", - "userAttributes": {"age": 30} - }' -``` - -What to verify: -- Different cmabUUID for each user -- Both requests succeed - -### Test 5: Multiple Flags - -If you have multiple CMAB-enabled flags, test requesting them together: - -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=flag1&keys=flag2" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_multi", - "userAttributes": {"age": 28} - }' -``` - -Expected response is an array: -```json -[ - { - "flagKey": "flag1", - "cmabUUID": "uuid-1", - ... - }, - { - "flagKey": "flag2", - "cmabUUID": "uuid-2", - ... - } -] -``` - -What to verify: -- Array response with multiple decisions -- Each CMAB flag has its own cmabUUID -- Non-CMAB flags won't have a UUID - -### Test 6: Decide Options Work with CMAB - -Request with decide options: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_options", - "userAttributes": {"age": 32}, - "decideOptions": ["INCLUDE_REASONS", "EXCLUDE_VARIABLES"] - }' -``` - -What to verify: -- reasons array is populated -- variables field is excluded or empty -- cmabUUID is still present - -### Test 7: Cache TTL Expiration - -To test this, you need to temporarily change the TTL in config.yaml: - -```yaml -cmab: - cache: - ttl: 1m # Short TTL for testing -``` - -Restart Agent, then: -1. Make a request and note the cmabUUID -2. Wait 65 seconds -3. Make the exact same request again - -What to verify: -- Second request returns a different cmabUUID (cache expired) - -### Test 8: Reset Endpoint - -The reset endpoint clears the client cache. This is useful for test isolation. - -Make a decision request: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_reset", - "userAttributes": {"age": 40} - }' -``` - -Note the cmabUUID, then call reset: -```bash -curl -X POST "http://localhost:8080/v1/reset" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" -``` - -Should return: -```json -{"result": true} -``` - -Make the same decision request again. - -What to verify: -- Reset returns {"result": true} -- After reset, the same request may get a different cmabUUID - -### Test 9: Forced Decisions Bypass CMAB - -Request with a forced decision: -```bash -curl -X POST "http://localhost:8080/v1/decide?keys=${FLAG_KEY}" \ - -H "X-Optimizely-SDK-Key: ${OPTIMIZELY_SDK_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "userId": "user_forced", - "userAttributes": {"age": 50}, - "forcedDecisions": [ - { - "flagKey": "'${FLAG_KEY}'", - "variationKey": "control" - } - ] - }' -``` - -What to verify: -- Returns the forced variationKey ("control") -- CMAB is bypassed (may not have cmabUUID) - -### Test 10: Configuration Changes - -Test that config changes take effect: - -**Test different timeout:** -Edit config.yaml: -```yaml -cmab: - requestTimeout: 30s -``` -Restart Agent and verify requests succeed with longer timeout. - -**Test smaller cache:** -Edit config.yaml: -```yaml -cmab: - cache: - size: 10 -``` -Restart Agent, make 15 different requests (different users), then repeat request #1. You might get a different UUID because the cache was full. - -**Test retry config:** -Edit config.yaml: -```yaml -cmab: - retryConfig: - maxRetries: 5 -``` -Restart Agent and check logs for retry behavior when the prediction endpoint is slow. - -## Configuration Reference - -Here's what you can configure in config.yaml: - -```yaml -cmab: - # Timeout for CMAB prediction API calls - requestTimeout: 10s - - cache: - # Cache type: "memory" or "redis" - type: "memory" - - # Maximum cached entries (memory cache only) - size: 1000 - - # How long cache entries live - ttl: 30m - - retryConfig: - # How many times to retry failed requests - maxRetries: 3 - - # Starting backoff duration - initialBackoff: 100ms - - # Maximum backoff duration - maxBackoff: 10s - - # Backoff multiplier (exponential) - backoffMultiplier: 2.0 -``` - -Settings you might want to test: - -| Setting | Default | Try Testing | What Changes | -|---------|---------|-------------|--------------| -| requestTimeout | 10s | 5s, 30s | Request timeout behavior | -| cache.size | 1000 | 10, 100 | Cache eviction | -| cache.ttl | 30m | 1m, 60m | Cache expiration | -| maxRetries | 3 | 0, 5 | Retry attempts | - -## What Good Results Look Like - -When CMAB is working correctly: - -**Decide responses contain:** -- cmabUUID field with a valid UUID -- variationKey from the CMAB decision -- ruleKey referencing the CMAB rule - -**Caching works:** -- Same requests return the same UUID -- Changing attributes gets a new decision -- Different users get separate cache entries - -**Configuration is respected:** -- TTL expires cache at the right time -- Timeout prevents hanging -- Retries happen as configured - -**Endpoints respond:** -- /v1/decide returns decisions -- /v1/reset clears the cache - -## Common Issues - -### No cmabUUID in the response - -Possible reasons: -- Flag isn't configured for CMAB in your project -- User doesn't qualify for the CMAB rule (check attributes) -- Prediction endpoint isn't accessible - -Debug by checking Agent logs: -```bash -tail -f agent.log | grep -i cmab -``` - -### Cache not working - -If every request returns a different UUID: -- Check if TTL is too short -- Verify you're not changing userID or attributes -- Check if cache size is too small - -Enable debug logging: -```bash -# In config.yaml -log: - level: debug - -# Restart and check logs -./bin/optimizely 2>&1 | grep cache -``` - -### Requests timeout - -Check: -- Is the prediction endpoint responsive? -- Is requestTimeout too short? - -Increase timeout temporarily: -```yaml -cmab: - requestTimeout: 30s -``` - -## Test Results Template - -Document your test results: - -``` -CMAB Testing Results -Date: __________ -Tester: __________ -Agent Version: __________ -SDK Key: __________ - -Test 1: Basic CMAB Decision [ ] Pass [ ] Fail -Test 2: Cache - Same Request [ ] Pass [ ] Fail -Test 3: Cache - Attribute Change [ ] Pass [ ] Fail -Test 4: Cache - Different Users [ ] Pass [ ] Fail -Test 5: Multiple Flags [ ] Pass [ ] Fail [ ] N/A -Test 6: Decide Options [ ] Pass [ ] Fail -Test 7: Cache TTL Expiration [ ] Pass [ ] Fail -Test 8: Reset Endpoint [ ] Pass [ ] Fail -Test 9: Forced Decisions [ ] Pass [ ] Fail -Test 10: Configuration Changes [ ] Pass [ ] Fail - -Overall: [ ] Pass [ ] Fail - -Notes: -_______________________________________ -``` - -## Quick Reference - -**API Endpoints:** -- POST /v1/decide - Get feature decisions (includes CMAB) -- POST /v1/reset - Clear client cache -- GET /health - Health check - -**Required Headers:** -``` -X-Optimizely-SDK-Key: your_sdk_key -Content-Type: application/json -``` - -**Sample Request:** -```json -{ - "userId": "test_user", - "userAttributes": { - "age": 25, - "location": "SF" - } -} -``` - -**CMAB Response Fields:** -```json -{ - "flagKey": "string", - "enabled": true, - "variationKey": "string", - "ruleKey": "string", - "cmabUUID": "550e8400-e29b-41d4-a716-446655440000", - "variables": {}, - "userContext": {} -} -``` From b042a30e440bba7afa89478db73a314ba3aa9126 Mon Sep 17 00:00:00 2001 From: Matjaz Pirnovar Date: Thu, 16 Oct 2025 14:50:40 -0700 Subject: [PATCH 4/4] Add test for CMAB endpoint environment variable to increase coverage --- pkg/optimizely/cache_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pkg/optimizely/cache_test.go b/pkg/optimizely/cache_test.go index 23292b4d..529bc697 100644 --- a/pkg/optimizely/cache_test.go +++ b/pkg/optimizely/cache_test.go @@ -20,6 +20,7 @@ package optimizely import ( "context" "fmt" + "os" "sync" "testing" "time" @@ -1107,6 +1108,37 @@ func (s *DefaultLoaderTestSuite) TestCMABWithExistingServices() { s.NotNil(client.odpCache, "ODP Cache should still be configured") } +func (s *DefaultLoaderTestSuite) TestCMABEndpointEnvironmentVariable() { + // Save original value and restore after test + originalEndpoint := os.Getenv("OPTIMIZELY_CMAB_PREDICTIONENDPOINT") + defer func() { + if originalEndpoint == "" { + os.Unsetenv("OPTIMIZELY_CMAB_PREDICTIONENDPOINT") + } else { + os.Setenv("OPTIMIZELY_CMAB_PREDICTIONENDPOINT", originalEndpoint) + } + }() + + // Set custom endpoint + testEndpoint := "https://test.prediction.endpoint.com/predict/%s" + os.Setenv("OPTIMIZELY_CMAB_PREDICTIONENDPOINT", testEndpoint) + + conf := config.ClientConfig{ + SdkKeyRegex: "sdkkey", + CMAB: config.CMABConfig{ + RequestTimeout: 5 * time.Second, + Cache: config.CMABCacheConfig{}, + RetryConfig: config.CMABRetryConfig{}, + }, + } + + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, nil, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + client, err := loader("sdkkey") + + s.NoError(err) + s.NotNil(client) +} + func TestDefaultLoaderTestSuite(t *testing.T) { suite.Run(t, new(DefaultLoaderTestSuite)) }