Use a separate ES index for each type #516

* Merge Behat search tests into SearchControllerTest * Adopt alias feature to enable non-destructive index population * Add index population deployment step * Tested with Elasticsearch 5 and 6
camdram · Nov 29, 2018 · 28b1047 · 28b1047
1 parent 590838e
commit 28b1047
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 99 deletions.
diff --git a/app/config/config_test.yml b/app/config/config_test.yml
@@ -1,6 +1,7 @@
 parameters:
     env(DATABASE_URL): "sqlite:///%kernel.cache_dir%/test.db"
     env(SEARCH_NAMESPACE): camdram_test
+    search_use_alias: false
 
 imports:
     - { resource: config.yml }

diff --git a/app/config/elasticsearch.yml b/app/config/elasticsearch.yml
@@ -1,10 +1,14 @@
+parameters:
+    search_use_alias: true
+
 fos_elastica:
     clients:
         default: { host: localhost, port: 9200 }
     indexes:
-        autocomplete:
-            index_name: '%env(SEARCH_NAMESPACE)%_autocomplete'
-            settings:
+        autocomplete_show:
+            index_name: '%env(SEARCH_NAMESPACE)%_autocomplete_show'
+            use_alias: "%search_use_alias%"
+            settings: &autocomplete_settings
                 index:
                     analysis:
                         tokenizer:
@@ -58,6 +62,11 @@ fos_elastica:
                         finder: ~
                         listener: { enabled: '%search_enable_listeners%' }
                     indexable_callback: isIndexable
+        autocomplete_society:
+            index_name: '%env(SEARCH_NAMESPACE)%_autocomplete_society'
+            use_alias: "%search_use_alias%"
+            settings: *autocomplete_settings
+            types:
                 society:
                     properties:
                         name: { analyzer: autocomplete, search_analyzer: search  }
@@ -69,6 +78,11 @@ fos_elastica:
                         provider: ~
                         finder: ~
                         listener: { enabled: '%search_enable_listeners%' }
+        autocomplete_venue:
+            index_name: '%env(SEARCH_NAMESPACE)%_autocomplete_venue'
+            use_alias: "%search_use_alias%"
+            settings: *autocomplete_settings
+            types:
                 venue:
                     properties:
                         name: { analyzer: autocomplete, search_analyzer: search }
@@ -79,6 +93,11 @@ fos_elastica:
                         provider: ~
                         finder: ~
                         listener: { enabled: '%search_enable_listeners%' }
+        autocomplete_person:
+            index_name: '%env(SEARCH_NAMESPACE)%_autocomplete_person'
+            use_alias: "%search_use_alias%"
+            settings: *autocomplete_settings
+            types:
                 person:
                     properties:
                         name: { analyzer: autocomplete, search_analyzer: search }

diff --git a/behat.yml b/behat.yml
@@ -9,10 +9,7 @@ default:
                 - UserContext:
                     entityManager: '@doctrine.orm.entity_manager'
                     encoderFactory: '@security.encoder_factory'
-                - SymfonyContext:
-                    listenersEnabled: '%%search_enable_listeners%%'
-                    elasticaResetter: '@fos_elastica.resetter'
-                    elasticaIndex: '@fos_elastica.index.autocomplete'
+                - SymfonyContext: ~
     extensions:
         Behat\Symfony2Extension: ~
         Behat\MinkExtension:

diff --git a/deploy.php b/deploy.php
@@ -71,6 +71,11 @@
     }
 })->desc('Refresh development database');
 
+// Elasticsearch tasks
+task('search:populate_index', function() {
+    run('{{bin/php}} {{bin/console}} fos:elastica:populate {{console_options}}');
+})->desc('Update Elasticsearch index');
+
 // Deployment Tasks
 
 task('deploy:validate', function() {
@@ -103,6 +108,7 @@
     'deploy:cache:warmup',
     'deploy:writable',
     'database:update',
+    'search:populate_index',
     'deploy:symlink',
     'deploy:unlock',
     'cleanup',

diff --git a/features/bootstrap/SymfonyContext.php b/features/bootstrap/SymfonyContext.php
@@ -23,20 +23,6 @@
  */
 class SymfonyContext extends RawMinkContext
 {
-    private $listenersEnabled;
-
-
-    private $elasticaResetter;
-
-    private $elasticaIndex;
-
-    public function __construct($listenersEnabled, Resetter $elasticaResetter, Index $elasticaIndex)
-    {
-        $this->listenersEnabled = $listenersEnabled;
-        $this->elasticaResetter = $elasticaResetter;
-        $this->elasticaIndex = $elasticaIndex;
-    }
-
     /**
      * @Given /^(.*) without redirection$/
      */
@@ -73,14 +59,6 @@ public function beforeScenario(BeforeScenarioScope $scope)
     {
         Time::mockDateTime(new \DateTime('2000-07-03 15:30:00'));
         StaticDriver::beginTransaction();
-
-        if ($this->listenersEnabled) {
-            $this->elasticaResetter->resetAllIndexes();
-        } 
-        elseif ($scope->getScenario()->hasTag('search')
-                || $scope->getFeature()->hasTag('search')) {
-            throw new PendingException('Elasticsearch not configured');
-        }
     }
 
     /**
@@ -90,12 +68,4 @@ public function afterScenario(AfterScenarioScope $scope)
     {
         StaticDriver::rollBack();
     }
-
-    /**
-     * @Given /^I refresh the search index$/
-     */
-    public function refreshSearchIndex()
-    {
-        $this->elasticaIndex->refresh();
-    }
 }
diff --git a/features/search.feature b/features/search.feature
diff --git a/src/Acts/CamdramBundle/Controller/AbstractRestController.php b/src/Acts/CamdramBundle/Controller/AbstractRestController.php
@@ -245,11 +245,10 @@ public function cgetAction(Request $request)
             $query->setFrom(($page-1)*$limit)->setSize($limit);
             //PHP_INT_MAX used because '_first' triggers an integer overflow in json_decode on 32 bit...
             $query->setSort([
-                'rank' => ['order' => 'desc', 'missing' => PHP_INT_MAX-1]
+                'rank' => ['order' => 'desc', 'unmapped_type' => 'long', 'missing' => PHP_INT_MAX-1]
             ]);
 
-            $search = $this->get('fos_elastica.index.autocomplete')->createSearch();
-            $search->addType($this->type);
+            $search = $this->get('fos_elastica.index.autocomplete_'.$this->type)->createSearch();
             $resultSet = $search->search($query);
 
             $data = [];

diff --git a/src/Acts/CamdramBundle/Controller/SearchController.php b/src/Acts/CamdramBundle/Controller/SearchController.php
@@ -35,12 +35,15 @@ public function searchAction(Request $request)
 
         $query = new Query($match);
         $query->setFrom(($page-1)*$limit)->setSize($limit);
-        //PHP_INT_MAX used because '_first' triggers an integer overflow in json_decode on 32 bit...
         $query->setSort([
-            'rank' => ['order' => 'desc', 'missing' => PHP_INT_MAX-1]
+            'rank' => ['order' => 'desc', 'unmapped_type' => 'long', 'missing' => '_first']
         ]);
 
-        $search = $this->get('fos_elastica.index.autocomplete')->createSearch();
+        $indexManager = $this->get('fos_elastica.index_manager');
+        $search = $indexManager->getIndex('autocomplete_show')->createSearch();
+        $search->addIndex($indexManager->getIndex('autocomplete_person')->getName());
+        $search->addIndex($indexManager->getIndex('autocomplete_society')->getName());
+        $search->addIndex($indexManager->getIndex('autocomplete_venue')->getName());
         $resultSet = $search->search($query);
 
         $data = [];

diff --git a/tests/Functional/SearchTest.php → ...undle/Controller/SearchControllerTest.php b/tests/Functional/SearchTest.php → ...undle/Controller/SearchControllerTest.php
@@ -13,7 +13,7 @@
 /**
  * @group search
  */
-class SearchTest extends WebTestCase
+class SearchControllerTest extends WebTestCase
 {
     /**
      * @var Symfony\Bundle\FrameworkBundle\Client
@@ -38,28 +38,19 @@ public function setUp()
         if (!$container->getParameter('search_enable_listeners')) {
             $this->markTestSkipped('search_enable_listeners is disabled');
         }
+
         $this->entityManager = $container->get('doctrine.orm.entity_manager');
         $container->get('fos_elastica.resetter')->resetAllIndexes();
-
-        $this->user = $this->createUser();
-    }
-
-    private function createUser()
-    {
-        $user = new User;
-        $user->setName("Test User")
-            ->setEmail("admin@camdram.net");
-        $this->entityManager->persist($user);
-        $this->entityManager->flush();
-
-        return $user;
     }
 
-    private function refreshIndex()
+    private function refreshIndexes()
     {
-        //Elasticsearch ordinarilytakes a few secs to update its index after a change.
-        //This ensures it's up to date before making assertions
-        $this->client->getKernel()->getContainer()->get('fos_elastica.index.autocomplete')->refresh();
+        //Elasticsearch ordinarily takes a few secs to update its indexes after a change.
+        //This ensures they're up to date before making assertions
+        $indexManager = $this->client->getKernel()->getContainer()->get('fos_elastica.index_manager');
+        foreach ($indexManager->getAllIndexes() as $index) {
+            $index->refresh();
+        };
     }
 
     private function createShow($name, $startDate, $flush = true)
@@ -155,7 +146,7 @@ public function testNoQuery()
     public function testAutocomplete()
     {
         $this->createShow('Test Show', '2000-01-01');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch('tes');
         $this->assertEquals(1, count($results));
@@ -200,11 +191,34 @@ public function testAutocomplete()
         $this->assertEquals('Test Show', $results[0]['name']);
     }
 
+    public function testHtmlView()
+    {
+        $this->createShow('Test Show 1', '2000-01-01');
+        $this->createShow('Test Show 2', '2000-02-01');
+        $this->createSociety('Test Society 1', 'soc1');
+        $this->createSociety('Test Society 2', 'scoc2');
+        $this->createVenue('Test Venue 1');
+        $this->createVenue('Test Venue 2');
+        $this->refreshIndexes();
+
+        $crawler = $this->client->request('GET', '/');
+        $form = $crawler->selectButton('Search')->form();
+        $form['q'] = 'test';
+        $crawler = $this->client->submit($form);
+
+        $this->assertEquals($crawler->filter('#content a:contains("Test Show 1")')->count(), 1);
+        $this->assertEquals($crawler->filter('#content a:contains("Test Show 2")')->count(), 1);
+        $this->assertEquals($crawler->filter('#content a:contains("Test Society 1")')->count(), 1);
+        $this->assertEquals($crawler->filter('#content a:contains("Test Society 2")')->count(), 1);
+        $this->assertEquals($crawler->filter('#content a:contains("Test Venue 1")')->count(), 1);
+        $this->assertEquals($crawler->filter('#content a:contains("Test Venue 2")')->count(), 1);
+    }
+
     public function testPunctuation()
     {
         $this->createShow("Journey's End", '2000-01-01');
         $this->createShow("Panto: Snow Queen", '2001-01-01');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch("journeys");
         $this->assertEquals("Journey's End", $results[0]['name']);
@@ -229,7 +243,7 @@ public function testAccents()
     {
         $this->createShow('Les Misérables', '2000-01-01');
         $this->createPerson('Zoë');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch("les mise");
         $this->assertEquals("Les Misérables", $results[0]['name']);
@@ -257,7 +271,7 @@ public function testUnicode()
     {
         $showName = '窦娥冤'; // "The Midsummer Snow" by Guan Hanqing
         $this->createShow($showName, '2000-01-01');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch('窦娥');
         $this->assertEquals($showName, $results[0]['name']);
@@ -277,7 +291,7 @@ public function testShowOrdering()
         $this->createShow('Test Show 2001', '2001-05-17');
         $this->createShow('Test Show 2012', '2012-01-02');
         $this->createShow('Test Show 1995', '1995-06-22');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch('test');
         $this->assertEquals(4, count($results));
@@ -295,7 +309,7 @@ public function testPagination()
             $this->createShow($name, $startAt, false);
         }
         $this->entityManager->flush();
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doPaginatedSearch('test', 1, 1);
         $this->assertEquals(1, count($results));
@@ -331,7 +345,7 @@ public function testLongName()
         $longName = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam viverra euismod justo sed malesuada. '
             . 'Fusce facilisis, neque nec faucibus blandit, sem orci auctor metus, ac luctus diam elit eget sapien.';
         $this->createShow($longName, '2000-01-01');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch($longName);
         $this->assertEquals($longName, $results[0]['name']);
@@ -340,7 +354,7 @@ public function testLongName()
     public function testPerson()
     {
         $this->createPerson('John Smith');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch('joh');
         $this->assertEquals('John Smith', $results[0]['name']);
@@ -359,7 +373,7 @@ public function testSociety()
     {
         $name = 'Cambridge University Amateur Dramatic Club';
         $this->createSociety($name, 'cuadc');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch('cam');
         $this->assertEquals($name, $results[0]['name']);
@@ -384,7 +398,7 @@ public function testSociety()
     public function testVenue()
     {
         $this->createVenue('ADC Theatre');
-        $this->refreshIndex();
+        $this->refreshIndexes();
 
         $results = $this->doSearch('ad');
         $this->assertEquals('ADC Theatre', $results[0]['name']);