Implement A/B testing support for Recommendations and Sorting command…

…s (RAD-1066)
lmc-eu · Apr 17, 2018 · 2de4da9 · 2de4da9
1 parent 3948c9b
commit 2de4da9
Show file tree

Hide file tree

Showing 8 changed files with 141 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,8 @@
 <!-- There is always Unreleased section on the top. Subsections (Added, Changed, Fixed, Removed) should be added as needed. -->
 
 ## Unreleased
+### Added
+- A/B testing support for recommendation and sorting requests.
 
 ## 1.4.0 - 2018-02-23
 ### Added

diff --git a/README.md b/README.md
@@ -251,6 +251,51 @@ $response = $matej->request()
     ->send();
 ```
 
+### A/B Testing support
+`Recommendation` and `Sorting` commands support optional A/B testing of various models. This has to be set up in Matej first,
+but once available, you can specify which model you want to use when requesting recommendations or sortings.
+
+This is available for `recommendation`, `sorting` and `campaign` requests:
+
+```php
+$recommendationCommand = UserRecommendation::create('user-id', 5, 'test-scenario', 1.0, 3600);
+$recommendationCommand->setModelName('alpha');
+
+$sortingCommand = Sorting::create('user-id', ['item-id-1', 'item-id-2', 'item-id-3']);
+$sortingCommand->setModelName('beta')
+
+$response = $matej->request()
+    ->recommendation($recommendationCommand)
+    ->send();
+
+$response = $matej->request()
+    ->sorting($sortingCommand)
+    ->send();
+
+$response = $matej->request()
+    ->campaign()
+    ->addRecommendation($recommendationCommand->setModelName('gamma'))
+    ->addSorting($sortingCommand->setModelName('delta'))
+    ->send();
+```
+
+If you don't provide any model name, the request will be sent without it, and Matej will use default model for your instance.
+
+Typically, you'd select a random sample of users, to which you'd present recommendations and sortings from second model. This way, implementation
+in your code should look similar to this:
+
+```php
+$recommendation = UserRecommendation::create('user-id', 5, 'test-scenario', 1.0, 3600);
+
+if ($session->isUserInBucketB()) {
+    $recommendation->setModelName('alpha');
+}
+
+$response = $matej->request()->recommendation($recommendation)->send();
+```
+
+Model names will be provided to you by LMC.
+
 ### Exceptions and error handling
 
 Exceptions are thrown only if the whole Request to Matej failed (when sending, decoding, authenticating etc.) or if

diff --git a/src/Model/Command/Sorting.php b/src/Model/Command/Sorting.php
@@ -14,6 +14,8 @@ class Sorting extends AbstractCommand implements UserAwareInterface
     private $userId;
     /** @var string[] */
     private $itemIds = [];
+    /** @var string|null */
+    private $modelName = null;
 
     private function __construct(string $userId, array $itemIds)
     {
@@ -31,6 +33,20 @@ public static function create(string $userId, array $itemIds): self
         return new static($userId, $itemIds);
     }
 
+    /**
+     * Set A/B model name
+     *
+     * @return $this
+     */
+    public function setModelName(string $modelName): self
+    {
+        Assertion::typeIdentifier($modelName);
+
+        $this->modelName = $modelName;
+
+        return $this;
+    }
+
     public function getUserId(): string
     {
         return $this->userId;
@@ -57,9 +73,15 @@ protected function getCommandType(): string
 
     protected function getCommandParameters(): array
     {
-        return [
+        $parameters = [
             'user_id' => $this->userId,
             'item_ids' => $this->itemIds,
         ];
+
+        if ($this->modelName !== null) {
+            $parameters['model_name'] = $this->modelName;
+        }
+
+        return $parameters;
     }
 }
diff --git a/src/Model/Command/UserRecommendation.php b/src/Model/Command/UserRecommendation.php
@@ -31,6 +31,8 @@ class UserRecommendation extends AbstractCommand implements UserAwareInterface
     private $minimalRelevance = self::MINIMAL_RELEVANCE_LOW;
     /** @var array */
     private $filters = ['valid_to >= NOW'];
+    /** @var string|null */
+    private $modelName = null;
 
     private function __construct(string $userId, int $count, string $scenario, float $rotationRate, int $rotationTime)
     {
@@ -121,6 +123,20 @@ public function setFilters(array $filters): self
         return $this;
     }
 
+    /***
+     * Set A/B model name
+     *
+     * @return $this
+     */
+    public function setModelName(string $modelName): self
+    {
+        Assertion::typeIdentifier($modelName);
+
+        $this->modelName = $modelName;
+
+        return $this;
+    }
+
     public function getUserId(): string
     {
         return $this->userId;
@@ -173,7 +189,7 @@ protected function getCommandType(): string
 
     protected function getCommandParameters(): array
     {
-        return [
+        $parameters = [
             'user_id' => $this->userId,
             'count' => $this->count,
             'scenario' => $this->scenario,
@@ -183,5 +199,11 @@ protected function getCommandParameters(): array
             'min_relevance' => $this->minimalRelevance,
             'filter' => $this->assembleFiltersString(),
         ];
+
+        if ($this->modelName !== null) {
+            $parameters['model_name'] = $this->modelName;
+        }
+
+        return $parameters;
     }
 }
diff --git a/tests/integration/RequestBuilder/RecommendationRequestBuilderTest.php b/tests/integration/RequestBuilder/RecommendationRequestBuilderTest.php
@@ -2,6 +2,7 @@
 
 namespace Lmc\Matej\IntegrationTests\RequestBuilder;
 
+use Lmc\Matej\Exception\RequestException;
 use Lmc\Matej\IntegrationTests\IntegrationTestCase;
 use Lmc\Matej\Model\Command\Interaction;
 use Lmc\Matej\Model\Command\UserMerge;
@@ -43,6 +44,19 @@ public function shouldExecuteRecommendationRequestWithUserMergeAndInteraction():
         $this->assertShorthandResponse($response, 'OK', 'OK', 'OK');
     }
 
+    /** @test */
+    public function shouldFailOnInvalidModelName(): void
+    {
+        $this->expectException(RequestException::class);
+        $this->expectExceptionCode(400);
+        $this->expectExceptionMessage('BAD REQUEST');
+
+        $this->createMatejInstance()
+            ->request()
+            ->recommendation($this->createRecommendationCommand('user-a')->setModelName('invalid-model-name'))
+            ->send();
+    }
+
     private function createRecommendationCommand(string $username): UserRecommendation
     {
         return UserRecommendation::create(

diff --git a/tests/integration/RequestBuilder/SortingRequestTest.php b/tests/integration/RequestBuilder/SortingRequestTest.php
@@ -2,6 +2,7 @@
 
 namespace Lmc\Matej\IntegrationTests\RequestBuilder;
 
+use Lmc\Matej\Exception\RequestException;
 use Lmc\Matej\IntegrationTests\IntegrationTestCase;
 use Lmc\Matej\Model\Command\Interaction;
 use Lmc\Matej\Model\Command\Sorting;
@@ -43,6 +44,19 @@ public function shouldExecuteSortingRequestWithUserMergeAndInteraction(): void
         $this->assertShorthandResponse($response, 'OK', 'OK', 'OK');
     }
 
+    /** @test */
+    public function shouldFailOnInvalidModelName(): void
+    {
+        $this->expectException(RequestException::class);
+        $this->expectExceptionCode(400);
+        $this->expectExceptionMessage('BAD REQUEST');
+
+        $this->createMatejInstance()
+            ->request()
+            ->sorting(Sorting::create('user-b', ['item-a', 'item-b', 'itemC-c'])->setModelName('invalid-model-name'))
+            ->send();
+    }
+
     private function assertShorthandResponse(SortingResponse $response, $interactionStatus, $userMergeStatus, $sortingStatus): void
     {
         $this->assertInstanceOf(CommandResponse::class, $response->getInteraction());

diff --git a/tests/unit/Model/Command/SortingTest.php b/tests/unit/Model/Command/SortingTest.php
@@ -11,26 +11,36 @@ public function shouldBeInstantiableViaNamedConstructor(): void
     {
         $userId = 'user-id';
         $itemIds = ['item-1', 'item-3', 'item-2'];
+        $modelName = 'test-model-name';
 
         $command = Sorting::create($userId, $itemIds);
         $this->assertSortingCommand($command, $userId, $itemIds);
+
+        $command->setModelName($modelName);
+        $this->assertSortingCommand($command, $userId, $itemIds, $modelName);
     }
 
     /**
      * Execute asserts against user merge command
      *
      * @param Sorting $command
      */
-    private function assertSortingCommand($command, string $userId, array $itemIds): void
+    private function assertSortingCommand($command, string $userId, array $itemIds, ?string $modelName = null): void
     {
+        $parameters = [
+            'user_id' => $userId,
+            'item_ids' => $itemIds,
+        ];
+
+        if ($modelName !== null) {
+            $parameters['model_name'] = $modelName;
+        }
+
         $this->assertInstanceOf(Sorting::class, $command);
         $this->assertSame(
             [
                 'type' => 'sorting',
-                'parameters' => [
-                    'user_id' => $userId,
-                    'item_ids' => $itemIds,
-                ],
+                'parameters' => $parameters,
             ],
             $command->jsonSerialize()
         );

diff --git a/tests/unit/Model/Command/UserRecommendationTest.php b/tests/unit/Model/Command/UserRecommendationTest.php
@@ -24,6 +24,7 @@ public function shouldBeInstantiableViaNamedConstructorWithDefaultValues(): void
                     'hard_rotation' => false,
                     'min_relevance' => UserRecommendation::MINIMAL_RELEVANCE_LOW,
                     'filter' => 'valid_to >= NOW',
+                    // intentionally no model name ==> should be absent when not used
                 ],
             ],
             $command->jsonSerialize()
@@ -39,12 +40,14 @@ public function shouldUseCustomParameters(): void
         $scenario = 'scenario-' . md5(microtime());
         $rotationRate = mt_rand() / mt_getrandmax();
         $rotationTime = random_int(1, 86400);
+        $modelName = 'test-model-' . md5(microtime());
 
         $command = UserRecommendation::create($userId, $count, $scenario, $rotationRate, $rotationTime);
 
         $command->setMinimalRelevance(UserRecommendation::MINIMAL_RELEVANCE_HIGH)
             ->enableHardRotation()
-            ->setFilters(['foo = bar', 'baz = ban']);
+            ->setFilters(['foo = bar', 'baz = ban'])
+            ->setModelName($modelName);
 
         $this->assertInstanceOf(UserRecommendation::class, $command);
         $this->assertSame(
@@ -59,6 +62,7 @@ public function shouldUseCustomParameters(): void
                     'hard_rotation' => true,
                     'min_relevance' => UserRecommendation::MINIMAL_RELEVANCE_HIGH,
                     'filter' => 'foo = bar and baz = ban',
+                    'model_name' => $modelName,
                 ],
             ],
             $command->jsonSerialize()