Integration (#13)

* FIX: Make null fields empty string, otherwise manticoresearch breaks * FIX: Ensure all fields are converted back to CamelCase * FIX: Stored fields now a thang * FIX: Filter highlighted fields * FIX: Coding standards, debug deletion * ENHANCEMENT: Multi fields now indexed * ENHANCEMENT: bigram indexing * FIX: Has many fields
gordonbanderson · Sep 7, 2020 · 756a5d2 · 756a5d2
1 parent cfd4be0
commit 756a5d2
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 12 deletions.
diff --git a/src/Service/BulkIndexer.php b/src/Service/BulkIndexer.php
@@ -10,6 +10,7 @@
 namespace Suilven\ManticoreSearch\Service;
 
 use SilverStripe\ORM\DataObject;
+use Suilven\FreeTextSearch\Factory\IndexerFactory;
 use Suilven\FreeTextSearch\Helper\IndexingHelper;
 
 /**
@@ -46,11 +47,26 @@ public function setIndex(string $newIndex): void
     public function addDataObject(DataObject $dataObject): void
     {
         $helper = new IndexingHelper();
-        $payload = $helper->getFieldsToIndex($dataObject);
+
+        $factory = new IndexerFactory();
+        $indexer = $factory->getIndexer();
+        $indexer->setIndexName($this->index);
+        $payload = $indexer->getIndexablePayload($dataObject);
         $toIndex = $payload[$this->index];
+
+        $keys = \array_keys($toIndex);
+        foreach ($keys as $key) {
+            if (!\is_null($toIndex[$key])) {
+                continue;
+            }
+
+            $toIndex[$key] = '';
+        }
+
         // @todo Fix indexing of parent id
         unset($toIndex['ParentID']);
         $this->bulkIndexData[$dataObject->ID] = $toIndex;
+
     }
 
 

diff --git a/src/Service/IndexCreator.php b/src/Service/IndexCreator.php
@@ -9,6 +9,7 @@
 
 namespace Suilven\ManticoreSearch\Service;
 
+use Suilven\FreeTextSearch\Indexes;
 use Suilven\FreeTextSearch\Types\FieldTypes;
 
 // @phpcs:disable Generic.Files.LineLength.TooLong
@@ -24,8 +25,12 @@ class IndexCreator extends \Suilven\FreeTextSearch\Base\IndexCreator implements
     public function createIndex(string $indexName): void
     {
         $fields = $this->getFields($indexName);
+        $storedFields = $this->getStoredFields($indexName);
         $specs = $this->getFieldSpecs($indexName);
 
+        \error_log('SPECS');
+        \print_r($specs);
+
         $columns = [];
         foreach ($fields as $field) {
             $fieldType = $specs[$field];
@@ -69,27 +74,36 @@ public function createIndex(string $indexName): void
 
             // override for Link, do not index it.  The storing of the Link URL is to save on database hierarchy
             // traversal when rendering search results
-            if ($field === 'Link') {
+            if ($field === 'Link' || \in_array($field, $storedFields, true)) {
                 $options = ['stored'];
             }
             $columns[$field] = ['type' => $indexType, 'options' => $options];
         }
 
 
+        // @todo Add has one
+
+        $indexes = new Indexes();
+        $index = $indexes->getIndex($indexName);
+        $mvaFields = $index->getHasManyFields();
+        error_log(print_r($mvaFields, true));
+
+        foreach(array_keys($mvaFields) as $mvaColumnName) {
+            $columns[$mvaColumnName] = ['type' => 'multi'];
+        }
 
         $client = new Client();
         $manticoreClient = $client->getConnection();
 
-
         $settings = [
             'rt_mem_limit' => '256M',
             'dict' => 'keywords',
             'min_infix_len' => 2,
             'html_strip' => 1,
+            'bigram_index' => 'both_freq'
         ];
 
 
-
         // drop index, and updating an existing one does not effect change
         $manticoreClient->indices()->drop(['index' => $indexName, 'body'=>['silent'=>true]]);
 

diff --git a/src/Service/Searcher.php b/src/Service/Searcher.php
@@ -34,18 +34,34 @@ public function search(?string $q): SearchResults
 
         $searcher = new Search($manticoreClient);
         $searcher->setIndex($this->indexName);
+
         $searcher->limit($this->pageSize);
         $offset=$this->pageSize * ($this->page-1);
         $searcher->offset($offset);
 
-        $manticoreResult = $searcher->search($q)->highlight(
+        $indexes = new Indexes();
+        $index = $indexes->getIndex($this->indexName);
+
+        $searcher->highlight(
             [],
             ['pre_tags' => '<b>', 'post_tags'=>'</b>']
-        )->get();
+        );
 
-        $indexes = new Indexes();
-        $index = $indexes->getIndex($this->indexName);
-        $fields = $index->getFields();
+        $manticoreResult = $searcher->search($q)->get();
+
+        $allFields = \array_merge(
+            $index->getFields(),
+            $index->getTokens(),
+            //$index->getHasManyFields(),
+            $index->getHasOneFields(),
+            $index->getStoredFields()
+        );
+
+
+        $hasManyFields = $index->getHasManyFields();
+        foreach(array_keys($hasManyFields) as $key) {
+            $allFields[] = $key;
+        }
 
         $ssResult = new ArrayList();
         while ($manticoreResult->valid()) {
@@ -55,9 +71,10 @@ public function search(?string $q): SearchResults
             $ssDataObject = new DataObject();
 
             $keys = \array_keys($source);
+
             foreach ($keys as $key) {
                 $keyname = $key;
-                foreach ($fields as $field) {
+                foreach ($allFields as $field) {
                     if (\strtolower($field) === $key) {
                         $keyname = $field;
 
@@ -81,13 +98,15 @@ public function search(?string $q): SearchResults
             $highlights = $hit->getHighlight();
             $highlightsSS = [];
 
+            $fieldsToHighlight = $index->getHighlightedFields();
+
             $keys = \array_keys($highlights);
             foreach ($keys as $key) {
-                if (!isset($highlights[$key])) {
+                if (!isset($highlights[$key]) || !\in_array($key, $fieldsToHighlight, true)) {
                     continue;
                 }
                 $keyname = $key;
-                foreach ($fields as $field) {
+                foreach ($allFields as $field) {
                     if (\strtolower($field) === $key) {
                         $keyname = $field;