Skip to content

Commit a1935cf

Browse files
degamidegami
committed
added search via ai embeddings
1 parent ba637d2 commit a1935cf

File tree

9 files changed

+518
-177
lines changed

9 files changed

+518
-177
lines changed

‎app/base/commands/Search/IndexerEmbeddings.php‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
<?php
2+
3+
/**
4+
* SiteBase
5+
* PHP Version 8.3
6+
*
7+
* @category CMS / Framework
8+
* @package Degami\Sitebase
9+
* @author Mirko De Grandis <degami@github.com>
10+
* @license MIT https://opensource.org/licenses/mit-license.php
11+
* @link https://github.com/degami/sitebase
12+
*/
13+
14+
namespace App\Base\Commands\Search;
15+
16+
use App\App;
17+
use App\Base\Abstracts\Commands\BaseCommand;
18+
use App\Base\Abstracts\Models\FrontendModel;
19+
use Degami\Basics\Exceptions\BasicException;
20+
use DI\DependencyException;
21+
use DI\NotFoundException;
22+
use Exception;
23+
use HaydenPierce\ClassFinder\ClassFinder;
24+
use Symfony\Component\Console\Input\InputInterface;
25+
use Symfony\Component\Console\Output\OutputInterface;
26+
use Symfony\Component\Console\Command\Command;
27+
use App\Base\Tools\Search\AIManager as AISearchManager;
28+
use Symfony\Component\Console\Input\InputArgument;
29+
30+
/**
31+
* Index embedding data for search engine
32+
*/
33+
class IndexerEmbeddings extends BaseCommand
34+
{
35+
/**
36+
* {@inheritdoc}
37+
*/
38+
protected function configure()
39+
{
40+
$this->setDescription('Index embedding data for search')
41+
->addArgument('llm', InputArgument::OPTIONAL, 'LLM to user', 'googlegemini');
42+
}
43+
44+
/**
45+
* {@inheritdoc}
46+
*
47+
* @param InputInterface $input
48+
* @param OutputInterface $output
49+
* @return void
50+
* @throws BasicException
51+
* @throws DependencyException
52+
* @throws NotFoundException
53+
* @throws \Exception
54+
*/
55+
protected function execute(InputInterface $input, OutputInterface $output) : int
56+
{
57+
if (!$this->getSearch()->isEnabled()) {
58+
$this->getIo()->error('Elasticsearch is not enabled');
59+
return Command::FAILURE;
60+
}
61+
62+
/** @var AISearchManager $embeddingManager */
63+
$embeddingManager = $this->containerMake(AISearchManager::class, [
64+
'llm' => $this->getAI()->getAIModel($input->getArgument('llm')),
65+
'model' => match ($input->getArgument('llm')) {
66+
'googlegemini' => 'text-embedding-004',
67+
'chatgpt' => 'text-embedding-3-small',
68+
'claude' => 'claude-2.0-embedding',
69+
'groq' => 'groq-vector-1',
70+
'mistral' => 'mistral-embedding-001',
71+
'perplexity' => 'perplexity-embedding-001',
72+
default => null,
73+
}
74+
]);
75+
76+
77+
if (!$embeddingManager->checkService()) {
78+
$this->getIo()->error('Service is not available');
79+
80+
return Command::FAILURE;
81+
}
82+
83+
if (!$embeddingManager->ensureIndex()) {
84+
$this->getIo()->error("Errors during index check");
85+
return Command::FAILURE;
86+
}
87+
88+
$classes = array_merge(
89+
ClassFinder::getClassesInNamespace(App::MODELS_NAMESPACE, ClassFinder::RECURSIVE_MODE),
90+
ClassFinder::getClassesInNamespace(App::BASE_MODELS_NAMESPACE, ClassFinder::RECURSIVE_MODE)
91+
);
92+
93+
$classes = array_filter($classes, function($className) {
94+
return is_subclass_of($className, FrontendModel::class) && $this->containerCall([$className, 'isIndexable']);
95+
});
96+
97+
if (!count($classes)) {
98+
$this->getIo()->error("No frontend classes found to index");
99+
return Command::FAILURE;
100+
}
101+
102+
$results = [];
103+
foreach ($classes as $modelClass) {
104+
$response = $embeddingManager->indexFrontendCollection($this->containerCall([$modelClass, 'getCollection']));
105+
foreach (($response['items'] ?? []) as $item) {
106+
if (isset($item['index']['result'])) {
107+
if (!isset($results[$item['index']['result']])) {
108+
$results[$item['index']['result']] = 0;
109+
}
110+
$results[$item['index']['result']]++;
111+
}
112+
}
113+
}
114+
115+
$this->renderTitle('Indexer results');
116+
$this->renderTable(array_keys($results), [$results]);
117+
118+
return Command::SUCCESS;
119+
}
120+
121+
122+
}

‎app/base/controllers/Frontend/Search.php‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
use DI\NotFoundException;
2222
use Symfony\Component\HttpFoundation\Response;
2323
use App\Base\Tools\Search\Manager as SearchManager;
24+
use App\Base\Tools\Search\AIManager as AISearchManager;
2425

2526
/**
2627
* Search page
@@ -87,7 +88,14 @@ protected function beforeRender() : BasePage|Response
8788
public function getTemplateData(): array
8889
{
8990
$page = $this->getRequest()->query->get('page') ?? 0;
90-
$search_result = $this->getSearchResult($this->getSearchQuery(), $page);
91+
$query = $this->getSearchQuery();
92+
$useAI = $this->getRequest()->query->get('ai') ?? false;
93+
94+
if ($useAI) {
95+
$search_result = $this->getAIsearchResult($query, 5);
96+
} else {
97+
$search_result = $this->getSearchResult($query, $page);
98+
}
9199

92100
return [
93101
'search_query' => $this->getSearchQuery(),
@@ -153,4 +161,49 @@ public function getCacheKey() : string
153161

154162
return $this->normalizeCacheKey($prefix . '.q='. $this->getSearchQuery().'.p='.$page);
155163
}
164+
165+
protected function getAIsearchResult(?string $search_query = null, int $k = 5, ?string $locale = null, $llmCode = 'googlegemini'): array
166+
{
167+
if ($search_query === null) {
168+
return ['total' => 0, 'docs' => []];
169+
}
170+
171+
/** @var AISearchManager $embeddingManager */
172+
$aiSearchManager = $this->containerMake(AISearchManager::class, [
173+
'llm' => $this->getAI()->getAIModel($llmCode),
174+
'model' => match ($llmCode) {
175+
'googlegemini' => 'text-embedding-004',
176+
'chatgpt' => 'text-embedding-3-small',
177+
'claude' => 'claude-2.0-embedding',
178+
'groq' => 'groq-vector-1',
179+
'mistral' => 'mistral-embedding-001',
180+
'perplexity' => 'perplexity-embedding-001',
181+
default => null,
182+
}
183+
]);
184+
185+
if ($locale === null) {
186+
$locale = $this->getCurrentLocale();
187+
}
188+
189+
$filters = [
190+
'locale' => $locale,
191+
'website_id' => $this->getSiteData()->getCurrentWebsiteId()
192+
];
193+
194+
$searchResult = $aiSearchManager->searchNearby($search_query, $k, $filters);
195+
196+
// Mappiamo i dati per essere compatibili con il template
197+
return [
198+
'total' => $searchResult['total'] ?? count($searchResult['docs']),
199+
'docs' => array_map(function($doc) {
200+
201+
return $this->getSearch()->getIndexDataForFrontendModel($this->containerCall(
202+
[$doc['data']['modelClass'], 'load'],
203+
['id' => $doc['data']['id']]
204+
))['_data'];
205+
206+
}, $searchResult['docs'] ?? [])
207+
];
208+
}
156209
}

‎app/base/event_listeners/GraphQL/SearchEventListener.php‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
use GraphQL\Type\Definition\ObjectType;
2121
use App\Base\Exceptions\NotFoundException;
2222
use GraphQL\Type\Definition\ResolveInfo;
23+
use App\Base\Tools\Search\AIManager as AISearchManager;
2324

2425
class SearchEventListener implements EventListenerInterface
2526
{
@@ -102,6 +103,72 @@ public function RegisterGraphQLQueryFields(Event $e)
102103
},
103104
];
104105
}
106+
107+
if (!isset($typesByName['NearbyItem'])) {
108+
$typesByName['NearbyItem'] = new ObjectType([
109+
'name' => 'NearbyItem',
110+
'fields' => [
111+
'id' => ['type' => Type::nonNull(Type::int())],
112+
'website_id' => ['type' => Type::nonNull(Type::int())],
113+
'locale' => ['type' => Type::string()],
114+
'modelClass' => ['type' => Type::string()],
115+
'type' => ['type' => Type::string()],
116+
'score' => ['type' => Type::float()],
117+
]
118+
]);
119+
}
120+
121+
if (!isset($queryFields['searchNearby'])) {
122+
$queryFields['searchNearby'] = [
123+
'type' => Type::listOf($typesByName['NearbyItem']),
124+
'args' => [
125+
'text' => ['type' => Type::nonNull(Type::string())],
126+
'k' => ['type' => Type::int()],
127+
'locale' => ['type' => Type::string()],
128+
'website_id' => ['type' => Type::int()],
129+
],
130+
'resolve' => function ($rootValue, $args, $context, ResolveInfo $info) use ($app) {
131+
$text = $args['text'];
132+
$llmCode = $args['llm'] ?? 'googlegemini';
133+
$k = $args['k'] ?? 5;
134+
135+
$filters = [];
136+
if (isset($args['locale'])) {
137+
$filters['locale'] = $args['locale'];
138+
}
139+
if (isset($args['website_id'])) {
140+
$filters['website_id'] = $args['website_id'];
141+
}
142+
143+
/** @var AISearchManager $embeddingManager */
144+
$aiSearchManager = App::getInstance()->containerMake(AISearchManager::class, [
145+
'llm' => App::getInstance()->getAI()->getAIModel($llmCode),
146+
'model' => match ($llmCode) {
147+
'googlegemini' => 'text-embedding-004',
148+
'chatgpt' => 'text-embedding-3-small',
149+
'claude' => 'claude-2.0-embedding',
150+
'groq' => 'groq-vector-1',
151+
'mistral' => 'mistral-embedding-001',
152+
'perplexity' => 'perplexity-embedding-001',
153+
default => null,
154+
}
155+
]);
156+
157+
$searchResult = $aiSearchManager->searchNearby($text, $k, $filters);
158+
159+
return array_map(function ($el) {
160+
return [
161+
'id' => $el['data']['id'],
162+
'website_id' => $el['data']['website_id'],
163+
'locale' => $el['data']['locale'],
164+
'modelClass' => $el['data']['modelClass'],
165+
'type' => $el['data']['type'],
166+
'score' => $el['score'],
167+
];
168+
}, $searchResult['docs'] ?? []);
169+
},
170+
];
171+
}
105172
}
106173

107174
/**

‎app/base/migrations/CreateApplicationLogTableMigration.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public function addDBTableDefinition(Table $table): Table
6262
->addColumn('created_at', 'TIMESTAMP', null, [], false, 'CURRENT_TIMESTAMP()')
6363
->addColumn('updated_at', 'TIMESTAMP', null, [], false, 'CURRENT_TIMESTAMP()')
6464
->addIndex(null, 'id', Index::TYPE_PRIMARY)
65-
->addForeignKey('fk_applicationlog_user_id', ['user_id'], 'user', ['id'])
65+
// ->addForeignKey('fk_applicationlog_user_id', ['user_id'], 'user', ['id'])
6666
->setAutoIncrementColumn('id');
6767

6868
return $table;

0 commit comments

Comments
 (0)