From 2de4da90eb63cbfba32fe2e8ce09f3e45a12e15e Mon Sep 17 00:00:00 2001 From: Pavel Ptacek Date: Wed, 11 Apr 2018 14:20:22 +0200 Subject: [PATCH] Implement A/B testing support for Recommendations and Sorting commands (RAD-1066) --- CHANGELOG.md | 2 + README.md | 45 +++++++++++++++++++ src/Model/Command/Sorting.php | 24 +++++++++- src/Model/Command/UserRecommendation.php | 24 +++++++++- .../RecommendationRequestBuilderTest.php | 14 ++++++ .../RequestBuilder/SortingRequestTest.php | 14 ++++++ tests/unit/Model/Command/SortingTest.php | 20 ++++++--- .../Model/Command/UserRecommendationTest.php | 6 ++- 8 files changed, 141 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64c65c0..49c8dfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ## Unreleased +### Added +- A/B testing support for recommendation and sorting requests. ## 1.4.0 - 2018-02-23 ### Added diff --git a/README.md b/README.md index 6f4d8e2..3c0c84f 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,51 @@ $response = $matej->request() ->send(); ``` +### A/B Testing support +`Recommendation` and `Sorting` commands support optional A/B testing of various models. This has to be set up in Matej first, +but once available, you can specify which model you want to use when requesting recommendations or sortings. + +This is available for `recommendation`, `sorting` and `campaign` requests: + +```php +$recommendationCommand = UserRecommendation::create('user-id', 5, 'test-scenario', 1.0, 3600); +$recommendationCommand->setModelName('alpha'); + +$sortingCommand = Sorting::create('user-id', ['item-id-1', 'item-id-2', 'item-id-3']); +$sortingCommand->setModelName('beta') + +$response = $matej->request() + ->recommendation($recommendationCommand) + ->send(); + +$response = $matej->request() + ->sorting($sortingCommand) + ->send(); + +$response = $matej->request() + ->campaign() + ->addRecommendation($recommendationCommand->setModelName('gamma')) + ->addSorting($sortingCommand->setModelName('delta')) + ->send(); +``` + +If you don't provide any model name, the request will be sent without it, and Matej will use default model for your instance. + +Typically, you'd select a random sample of users, to which you'd present recommendations and sortings from second model. This way, implementation +in your code should look similar to this: + +```php +$recommendation = UserRecommendation::create('user-id', 5, 'test-scenario', 1.0, 3600); + +if ($session->isUserInBucketB()) { + $recommendation->setModelName('alpha'); +} + +$response = $matej->request()->recommendation($recommendation)->send(); +``` + +Model names will be provided to you by LMC. + ### Exceptions and error handling Exceptions are thrown only if the whole Request to Matej failed (when sending, decoding, authenticating etc.) or if diff --git a/src/Model/Command/Sorting.php b/src/Model/Command/Sorting.php index 99bf96c..0f2e62b 100644 --- a/src/Model/Command/Sorting.php +++ b/src/Model/Command/Sorting.php @@ -14,6 +14,8 @@ class Sorting extends AbstractCommand implements UserAwareInterface private $userId; /** @var string[] */ private $itemIds = []; + /** @var string|null */ + private $modelName = null; private function __construct(string $userId, array $itemIds) { @@ -31,6 +33,20 @@ public static function create(string $userId, array $itemIds): self return new static($userId, $itemIds); } + /** + * Set A/B model name + * + * @return $this + */ + public function setModelName(string $modelName): self + { + Assertion::typeIdentifier($modelName); + + $this->modelName = $modelName; + + return $this; + } + public function getUserId(): string { return $this->userId; @@ -57,9 +73,15 @@ protected function getCommandType(): string protected function getCommandParameters(): array { - return [ + $parameters = [ 'user_id' => $this->userId, 'item_ids' => $this->itemIds, ]; + + if ($this->modelName !== null) { + $parameters['model_name'] = $this->modelName; + } + + return $parameters; } } diff --git a/src/Model/Command/UserRecommendation.php b/src/Model/Command/UserRecommendation.php index 32434ae..a3697d5 100644 --- a/src/Model/Command/UserRecommendation.php +++ b/src/Model/Command/UserRecommendation.php @@ -31,6 +31,8 @@ class UserRecommendation extends AbstractCommand implements UserAwareInterface private $minimalRelevance = self::MINIMAL_RELEVANCE_LOW; /** @var array */ private $filters = ['valid_to >= NOW']; + /** @var string|null */ + private $modelName = null; private function __construct(string $userId, int $count, string $scenario, float $rotationRate, int $rotationTime) { @@ -121,6 +123,20 @@ public function setFilters(array $filters): self return $this; } + /*** + * Set A/B model name + * + * @return $this + */ + public function setModelName(string $modelName): self + { + Assertion::typeIdentifier($modelName); + + $this->modelName = $modelName; + + return $this; + } + public function getUserId(): string { return $this->userId; @@ -173,7 +189,7 @@ protected function getCommandType(): string protected function getCommandParameters(): array { - return [ + $parameters = [ 'user_id' => $this->userId, 'count' => $this->count, 'scenario' => $this->scenario, @@ -183,5 +199,11 @@ protected function getCommandParameters(): array 'min_relevance' => $this->minimalRelevance, 'filter' => $this->assembleFiltersString(), ]; + + if ($this->modelName !== null) { + $parameters['model_name'] = $this->modelName; + } + + return $parameters; } } diff --git a/tests/integration/RequestBuilder/RecommendationRequestBuilderTest.php b/tests/integration/RequestBuilder/RecommendationRequestBuilderTest.php index f69c1e8..27f534e 100644 --- a/tests/integration/RequestBuilder/RecommendationRequestBuilderTest.php +++ b/tests/integration/RequestBuilder/RecommendationRequestBuilderTest.php @@ -2,6 +2,7 @@ namespace Lmc\Matej\IntegrationTests\RequestBuilder; +use Lmc\Matej\Exception\RequestException; use Lmc\Matej\IntegrationTests\IntegrationTestCase; use Lmc\Matej\Model\Command\Interaction; use Lmc\Matej\Model\Command\UserMerge; @@ -43,6 +44,19 @@ public function shouldExecuteRecommendationRequestWithUserMergeAndInteraction(): $this->assertShorthandResponse($response, 'OK', 'OK', 'OK'); } + /** @test */ + public function shouldFailOnInvalidModelName(): void + { + $this->expectException(RequestException::class); + $this->expectExceptionCode(400); + $this->expectExceptionMessage('BAD REQUEST'); + + $this->createMatejInstance() + ->request() + ->recommendation($this->createRecommendationCommand('user-a')->setModelName('invalid-model-name')) + ->send(); + } + private function createRecommendationCommand(string $username): UserRecommendation { return UserRecommendation::create( diff --git a/tests/integration/RequestBuilder/SortingRequestTest.php b/tests/integration/RequestBuilder/SortingRequestTest.php index aa618d9..f28ab36 100644 --- a/tests/integration/RequestBuilder/SortingRequestTest.php +++ b/tests/integration/RequestBuilder/SortingRequestTest.php @@ -2,6 +2,7 @@ namespace Lmc\Matej\IntegrationTests\RequestBuilder; +use Lmc\Matej\Exception\RequestException; use Lmc\Matej\IntegrationTests\IntegrationTestCase; use Lmc\Matej\Model\Command\Interaction; use Lmc\Matej\Model\Command\Sorting; @@ -43,6 +44,19 @@ public function shouldExecuteSortingRequestWithUserMergeAndInteraction(): void $this->assertShorthandResponse($response, 'OK', 'OK', 'OK'); } + /** @test */ + public function shouldFailOnInvalidModelName(): void + { + $this->expectException(RequestException::class); + $this->expectExceptionCode(400); + $this->expectExceptionMessage('BAD REQUEST'); + + $this->createMatejInstance() + ->request() + ->sorting(Sorting::create('user-b', ['item-a', 'item-b', 'itemC-c'])->setModelName('invalid-model-name')) + ->send(); + } + private function assertShorthandResponse(SortingResponse $response, $interactionStatus, $userMergeStatus, $sortingStatus): void { $this->assertInstanceOf(CommandResponse::class, $response->getInteraction()); diff --git a/tests/unit/Model/Command/SortingTest.php b/tests/unit/Model/Command/SortingTest.php index b1aff81..f99eda0 100644 --- a/tests/unit/Model/Command/SortingTest.php +++ b/tests/unit/Model/Command/SortingTest.php @@ -11,9 +11,13 @@ public function shouldBeInstantiableViaNamedConstructor(): void { $userId = 'user-id'; $itemIds = ['item-1', 'item-3', 'item-2']; + $modelName = 'test-model-name'; $command = Sorting::create($userId, $itemIds); $this->assertSortingCommand($command, $userId, $itemIds); + + $command->setModelName($modelName); + $this->assertSortingCommand($command, $userId, $itemIds, $modelName); } /** @@ -21,16 +25,22 @@ public function shouldBeInstantiableViaNamedConstructor(): void * * @param Sorting $command */ - private function assertSortingCommand($command, string $userId, array $itemIds): void + private function assertSortingCommand($command, string $userId, array $itemIds, ?string $modelName = null): void { + $parameters = [ + 'user_id' => $userId, + 'item_ids' => $itemIds, + ]; + + if ($modelName !== null) { + $parameters['model_name'] = $modelName; + } + $this->assertInstanceOf(Sorting::class, $command); $this->assertSame( [ 'type' => 'sorting', - 'parameters' => [ - 'user_id' => $userId, - 'item_ids' => $itemIds, - ], + 'parameters' => $parameters, ], $command->jsonSerialize() ); diff --git a/tests/unit/Model/Command/UserRecommendationTest.php b/tests/unit/Model/Command/UserRecommendationTest.php index 3beac4a..4e609ed 100644 --- a/tests/unit/Model/Command/UserRecommendationTest.php +++ b/tests/unit/Model/Command/UserRecommendationTest.php @@ -24,6 +24,7 @@ public function shouldBeInstantiableViaNamedConstructorWithDefaultValues(): void 'hard_rotation' => false, 'min_relevance' => UserRecommendation::MINIMAL_RELEVANCE_LOW, 'filter' => 'valid_to >= NOW', + // intentionally no model name ==> should be absent when not used ], ], $command->jsonSerialize() @@ -39,12 +40,14 @@ public function shouldUseCustomParameters(): void $scenario = 'scenario-' . md5(microtime()); $rotationRate = mt_rand() / mt_getrandmax(); $rotationTime = random_int(1, 86400); + $modelName = 'test-model-' . md5(microtime()); $command = UserRecommendation::create($userId, $count, $scenario, $rotationRate, $rotationTime); $command->setMinimalRelevance(UserRecommendation::MINIMAL_RELEVANCE_HIGH) ->enableHardRotation() - ->setFilters(['foo = bar', 'baz = ban']); + ->setFilters(['foo = bar', 'baz = ban']) + ->setModelName($modelName); $this->assertInstanceOf(UserRecommendation::class, $command); $this->assertSame( @@ -59,6 +62,7 @@ public function shouldUseCustomParameters(): void 'hard_rotation' => true, 'min_relevance' => UserRecommendation::MINIMAL_RELEVANCE_HIGH, 'filter' => 'foo = bar and baz = ban', + 'model_name' => $modelName, ], ], $command->jsonSerialize()