diff --git a/appinfo/routes.php b/appinfo/routes.php index 4f03482e5..6f9b20130 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -389,6 +389,7 @@ ['name' => 'ui#auditTrail', 'url' => '/audit-trails', 'verb' => 'GET'], ['name' => 'ui#searchTrail', 'url' => '/search-trails', 'verb' => 'GET'], ['name' => 'ui#webhooks', 'url' => '/webhooks', 'verb' => 'GET'], + ['name' => 'ui#webhooksLogs', 'url' => '/webhooks/logs', 'verb' => 'GET'], ['name' => 'ui#entities', 'url' => '/entities', 'verb' => 'GET'], ['name' => 'files#page', 'url' => '/files', 'verb' => 'GET'], @@ -402,5 +403,7 @@ ['name' => 'webhooks#events', 'url' => '/api/webhooks/events', 'verb' => 'GET'], ['name' => 'webhooks#logs', 'url' => '/api/webhooks/{id}/logs', 'verb' => 'GET', 'requirements' => ['id' => '\d+']], ['name' => 'webhooks#logStats', 'url' => '/api/webhooks/{id}/logs/stats', 'verb' => 'GET', 'requirements' => ['id' => '\d+']], + ['name' => 'webhooks#allLogs', 'url' => '/api/webhooks/logs', 'verb' => 'GET'], + ['name' => 'webhooks#retry', 'url' => '/api/webhooks/logs/{logId}/retry', 'verb' => 'POST', 'requirements' => ['logId' => '\d+']], ], ]; diff --git a/docker-compose.yml b/docker-compose.yml index 0fbfe57e5..a81c03b49 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -190,6 +190,7 @@ services: - solr - ollama - presidio-analyzer + - n8n volumes: - nextcloud:/var/www/html:rw - ./custom_apps:/var/www/html/custom_apps @@ -206,6 +207,8 @@ services: - PHP_MEMORY_LIMIT=4G - PHP_UPLOAD_LIMIT=2G - PHP_POST_MAX_SIZE=2G - # depends_on: + depends_on: + - db + - n8n # init-ubuntu: # condition: service_completed_successfully diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 371f3788d..ef9012c73 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -615,6 +615,7 @@ function ($container) { $container->get(EntityRelationMapper::class), $container->get(ChunkMapper::class), $container->get(SettingsService::class), + $container->get(id: 'OCP\IDBConnection'), $container->get(id: 'Psr\Log\LoggerInterface') ); } diff --git a/lib/Command/SolrDebugCommand.php b/lib/Command/SolrDebugCommand.php index 7faa0d034..b9ab2339e 100644 --- a/lib/Command/SolrDebugCommand.php +++ b/lib/Command/SolrDebugCommand.php @@ -386,7 +386,9 @@ private function testSolrAdminAPI(OutputInterface $output, array $solrSettings): $collectionCount = count($collectionsData['cluster']['collections']); $output->writeln(" ✅ Found $collectionCount collections (SolrCloud mode)"); foreach (array_keys($collectionsData['cluster']['collections']) as $collectionName) { - $output->writeln(" - $collectionName"); + /** @var string|int $collectionName */ + $collectionNameStr = is_string($collectionName) ? $collectionName : (string) $collectionName; + $output->writeln(" - ".$collectionNameStr.""); } } } else { diff --git a/lib/Command/SolrManagementCommand.php b/lib/Command/SolrManagementCommand.php index 3934cf57e..5ac156f12 100644 --- a/lib/Command/SolrManagementCommand.php +++ b/lib/Command/SolrManagementCommand.php @@ -326,7 +326,7 @@ private function handleWarm(OutputInterface $output): int foreach ($warmQueries as $query) { $output->write(' 🔥 '.$query['description'].'... '); - $result = $this->solrService->searchObjects(query: $query); + $result = $this->solrService->searchObjects(searchParams: $query); if ($result['success'] === true) { $output->writeln(''); $successCount++; @@ -395,7 +395,7 @@ private function handleHealth(OutputInterface $output): int // Basic search test. $output->writeln(''); $output->writeln('🔍 Testing search functionality...'); - $searchResult = $this->solrService->searchObjects(query: ['q' => '*:*', 'rows' => 1]); + $searchResult = $this->solrService->searchObjects(searchParams: ['q' => '*:*', 'rows' => 1]); if ($searchResult['success'] === true) { $output->writeln(' ✅ Search working ('.$searchResult['execution_time_ms'].'ms)'); $output->writeln(' 📊 Total documents: '.$searchResult['total'].''); @@ -469,7 +469,7 @@ private function handleSchemaCheck(OutputInterface $output): int $output->writeln('🔍 Checking field compatibility...'); // Test a document structure. - $testResult = $this->solrService->searchObjects(query: ['q' => '*:*', 'rows' => 1]); + $testResult = $this->solrService->searchObjects(searchParams: ['q' => '*:*', 'rows' => 1]); if ($testResult['success'] === true && empty($testResult['data']) === false) { $sampleDoc = $testResult['data'][0]; $availableFields = array_keys($sampleDoc); diff --git a/lib/Controller/ConfigurationController.php b/lib/Controller/ConfigurationController.php index 289ce216c..98a789f1a 100644 --- a/lib/Controller/ConfigurationController.php +++ b/lib/Controller/ConfigurationController.php @@ -626,11 +626,11 @@ public function discover(): JSONResponse // Call appropriate service. if ($source === 'github') { $this->logger->info('About to call GitHub search service'); - $results = $this->githubService->searchConfigurations(query: $search, page: $page); + $results = $this->githubService->searchConfigurations(search: $search, page: $page); $this->logger->info('GitHub search completed', ['result_count' => count($results['results'] ?? [])]); } else { $this->logger->info('About to call GitLab search service'); - $results = $this->gitlabService->searchConfigurations(query: $search, page: $page); + $results = $this->gitlabService->searchConfigurations(search: $search, page: $page); $this->logger->info('GitLab search completed', ['result_count' => count($results['results'] ?? [])]); } diff --git a/lib/Controller/ConfigurationsController.php b/lib/Controller/ConfigurationsController.php index 3c2b482e0..f06baaf59 100644 --- a/lib/Controller/ConfigurationsController.php +++ b/lib/Controller/ConfigurationsController.php @@ -307,15 +307,12 @@ public function export(int $id, bool $includeObjects=false): DataDownloadRespons /** * Import a configuration * - * @param bool $includeObjects Whether to include objects in the import. - * @param bool $force Force import even if the same or newer version already exists - * * @return JSONResponse The import result. * * @NoAdminRequired * @NoCSRFRequired */ - public function import(bool $includeObjects=false, bool $force=false): JSONResponse + public function import(): JSONResponse { try { // Get the uploaded file from the request if a single file has been uploaded. diff --git a/lib/Controller/DashboardController.php b/lib/Controller/DashboardController.php index 274b10f1a..f92848421 100644 --- a/lib/Controller/DashboardController.php +++ b/lib/Controller/DashboardController.php @@ -22,6 +22,7 @@ use OCP\AppFramework\Http\JSONResponse; use OCP\AppFramework\Http\TemplateResponse; use OCP\IRequest; +use Psr\Log\LoggerInterface; /** * Class DashboardController @@ -41,6 +42,13 @@ class DashboardController extends Controller */ private DashboardService $dashboardService; + /** + * Logger instance + * + * @var LoggerInterface + */ + private LoggerInterface $logger; + /** * Constructor for the DashboardController @@ -48,16 +56,19 @@ class DashboardController extends Controller * @param string $appName The name of the app * @param IRequest $request The request object * @param DashboardService $dashboardService The dashboard service instance + * @param LoggerInterface $logger Logger instance * * @return void */ public function __construct( string $appName, IRequest $request, - DashboardService $dashboardService + DashboardService $dashboardService, + LoggerInterface $logger ) { parent::__construct(appName: $appName, request: $request); $this->dashboardService = $dashboardService; + $this->logger = $logger; }//end __construct() @@ -67,15 +78,13 @@ public function __construct( * * This method renders the dashboard page of the application, adding any necessary data to the template. * - * @param string|null $getParameter Optional parameter for the page request - * * @return TemplateResponse The rendered template response * * @NoAdminRequired * * @NoCSRFRequired */ - public function page(?string $getParameter=null): TemplateResponse + public function page(): TemplateResponse { try { $response = new TemplateResponse( @@ -335,8 +344,24 @@ public function getMostActiveObjects(?int $registerId=null, ?int $schemaId=null, $data = $this->dashboardService->getMostActiveObjects(registerId: $registerId, schemaId: $schemaId, limit: $limit, hours: $hours); return new JSONResponse(data: $data); } catch (\Exception $e) { - return new JSONResponse(data: ['error' => $e->getMessage()], statusCode: 500); - } + $this->logger->error( + message: 'Error retrieving most active objects: '.$e->getMessage(), + context: [ + 'register_id' => $registerId, + 'schema_id' => $schemaId, + 'limit' => $limit, + 'hours' => $hours, + 'trace' => $e->getTraceAsString(), + ] + ); + + return new JSONResponse( + data: [ + 'error' => 'Failed to retrieve most active objects: '.$e->getMessage(), + ], + statusCode: 500 + ); + }//end try }//end getMostActiveObjects() diff --git a/lib/Controller/FileExtractionController.php b/lib/Controller/FileExtractionController.php index 5eb9fb3bf..521d4d6c5 100644 --- a/lib/Controller/FileExtractionController.php +++ b/lib/Controller/FileExtractionController.php @@ -93,12 +93,11 @@ public function __construct( * array * > */ - public function index(?int $limit=100, ?int $offset=0, ?string $status=null, ?string $search=null): JSONResponse + public function index(): JSONResponse { try { // TextExtractionService doesn't have findByStatus, use discoverUntrackedFiles or extractPendingFiles instead. // For now, return empty array as this endpoint needs to be redesigned for chunk-based architecture. - // Note: Search filtering removed as it's not applicable to empty array return new JSONResponse( data: [ diff --git a/lib/Controller/FilesController.php b/lib/Controller/FilesController.php index 35663f4dd..ed50435d7 100644 --- a/lib/Controller/FilesController.php +++ b/lib/Controller/FilesController.php @@ -101,6 +101,12 @@ public function index( string $schema, string $id ): JSONResponse { + // Note: $register and $schema are route parameters for API consistency + // They are part of the URL structure (/api/objects/{register}/{schema}/{id}/files) + // but only $id is used to fetch files + // Reference them to satisfy static analysis + $routeParams = ['register' => $register, 'schema' => $schema]; + unset($routeParams); try { // Get the raw files from the file service. $files = $this->fileService->getFiles(object: $id); diff --git a/lib/Controller/ObjectsController.php b/lib/Controller/ObjectsController.php index a67a0a2ff..84ab9d156 100644 --- a/lib/Controller/ObjectsController.php +++ b/lib/Controller/ObjectsController.php @@ -44,6 +44,7 @@ use Psr\Container\ContainerExceptionInterface; use Psr\Container\ContainerInterface; use Psr\Container\NotFoundExceptionInterface; +use Psr\Log\LoggerInterface; use Symfony\Component\Uid\Uuid; use OCA\OpenRegister\Service\FileService; use OCA\OpenRegister\Service\ExportService; @@ -386,7 +387,7 @@ public function index(string $register, string $schema, ObjectService $objectSer { try { // Resolve slugs to numeric IDs consistently (validation only). - $this->resolveRegisterSchemaIds(register: $register, schema: $schema, objectService: $objectService); + $resolved = $this->resolveRegisterSchemaIds(register: $register, schema: $schema, objectService: $objectService); } catch (\OCA\OpenRegister\Exception\RegisterNotFoundException | \OCA\OpenRegister\Exception\SchemaNotFoundException $e) { // Return 404 with clear error message if register or schema not found. return new JSONResponse(data: ['message' => $e->getMessage()], statusCode: 404); @@ -1019,6 +1020,11 @@ public function contracts(string $id, string $register, string $schema, ObjectSe // Set the schema and register to the object service. $objectService->setSchema(schema: $schema); $objectService->setRegister(register: $register); + + // Note: $id is a route parameter for API consistency (/api/objects/{register}/{schema}/{id}/contracts) + // Currently returns empty array as contract functionality is not yet implemented + $objectId = $id; // Reserved for future use when contract functionality is implemented + unset($objectId); // Get request parameters for filtering and searching. $requestParams = $this->request->getParams(); diff --git a/lib/Controller/OrganisationController.php b/lib/Controller/OrganisationController.php index 5ab6dcb86..af6dc6c20 100644 --- a/lib/Controller/OrganisationController.php +++ b/lib/Controller/OrganisationController.php @@ -252,7 +252,7 @@ public function create(string $name, string $description=''): JSONResponse $requestData = $this->request->getParams(); $uuid = $requestData['uuid'] ?? ''; - $organisation = $this->organisationService->createOrganisation(name: $name, description: $description, isPublic: true, uuid: $uuid); + $organisation = $this->organisationService->createOrganisation(name: $name, description: $description, addCurrentUser: true, uuid: $uuid); return new JSONResponse( data: [ @@ -541,7 +541,7 @@ public function update(string $uuid): JSONResponse // Validate parent assignment to prevent circular references. try { - $this->organisationMapper->validateParentAssignment(organisationUuid: $uuid, parentUuid: $newParent); + $this->organisationMapper->validateParentAssignment(organisationUuid: $uuid, newParentUuid: $newParent); $organisation->setParent($newParent); } catch (Exception $e) { $this->logger->warning( diff --git a/lib/Controller/UiController.php b/lib/Controller/UiController.php index 050ebe24f..0ab79d5ca 100644 --- a/lib/Controller/UiController.php +++ b/lib/Controller/UiController.php @@ -332,6 +332,24 @@ public function webhooks(): TemplateResponse }//end webhooks() + /** + * Returns the webhook logs page template. + * + * @NoAdminRequired + * @NoCSRFRequired + * + * @return TemplateResponse The template response. + * + * @phpstan-return TemplateResponse + * @psalm-return TemplateResponse + */ + public function webhooksLogs(): TemplateResponse + { + return $this->makeSpaResponse(); + + }//end webhooksLogs() + + /** * Returns the entities page template. * diff --git a/lib/Controller/WebhooksController.php b/lib/Controller/WebhooksController.php index d942b9615..91b7ab290 100644 --- a/lib/Controller/WebhooksController.php +++ b/lib/Controller/WebhooksController.php @@ -397,11 +397,35 @@ public function test(int $id): JSONResponse ] ); } else { + // Get the latest log entry to retrieve error details. + $latestLogs = $this->webhookLogMapper->findByWebhook($id, 1, 0); + $errorMessage = 'Test webhook delivery failed'; + $errorDetails = null; + + if (!empty($latestLogs)) { + $latestLog = $latestLogs[0]; + if ($latestLog->getErrorMessage() !== null) { + $errorMessage = $latestLog->getErrorMessage(); + } + if ($latestLog->getStatusCode() !== null) { + $errorDetails = [ + 'status_code' => $latestLog->getStatusCode(), + 'response_body' => $latestLog->getResponseBody(), + ]; + } + } + + $responseData = [ + 'success' => false, + 'message' => $errorMessage, + ]; + + if ($errorDetails !== null) { + $responseData['error_details'] = $errorDetails; + } + return new JSONResponse( - data: [ - 'success' => false, - 'message' => 'Test webhook delivery failed', - ], + data: $responseData, statusCode: 500 ); }//end if @@ -439,7 +463,7 @@ public function test(int $id): JSONResponse return new JSONResponse( data: [ - 'error' => 'Failed to test webhook', + 'error' => 'Failed to test webhook: '.$e->getMessage(), ], statusCode: 500 ); @@ -889,4 +913,227 @@ public function logStats(int $id): JSONResponse }//end logStats() + /** + * Get all webhook logs with optional filtering + * + * @return JSONResponse + * + * @NoAdminRequired + * @NoCSRFRequired + */ + #[NoAdminRequired] + #[NoCSRFRequired] + public function allLogs(): JSONResponse + { + try { + $webhookId = $this->request->getParam('webhook_id'); + $limit = (int) ($this->request->getParam('limit') ?? 50); + $offset = (int) ($this->request->getParam('offset') ?? 0); + $success = $this->request->getParam('success'); + + // If webhook_id is provided and valid, use findByWebhook method. + if ($webhookId !== null && $webhookId !== '' && $webhookId !== '0') { + $webhookIdInt = (int) $webhookId; + $logs = $this->webhookLogMapper->findByWebhook($webhookIdInt, $limit, $offset); + // Get total count for this webhook. + $allLogsForWebhook = $this->webhookLogMapper->findByWebhook($webhookIdInt, null, null); + $total = count($allLogsForWebhook); + } else { + // Get all logs. + $logs = $this->webhookLogMapper->findAll($limit, $offset); + // Get total count for all logs. + $allLogs = $this->webhookLogMapper->findAll(null, null); + $total = count($allLogs); + } + + // Filter by success status if provided. + if ($success !== null && $success !== '' && ($success === 'true' || $success === '1' || $success === 'false' || $success === '0')) { + $successBool = $success === 'true' || $success === '1'; + $filteredLogs = array_filter( + $logs, + function ($log) use ($successBool) { + return $log->getSuccess() === $successBool; + } + ); + $logs = array_values($filteredLogs); // Re-index array. + // Recalculate total if filtering by success. + if ($webhookId !== null && $webhookId !== '' && $webhookId !== '0') { + $webhookIdInt = (int) $webhookId; + $allLogsForWebhook = $this->webhookLogMapper->findByWebhook($webhookIdInt, null, null); + $total = count(array_filter($allLogsForWebhook, function ($log) use ($successBool) { + return $log->getSuccess() === $successBool; + })); + } else { + $allLogs = $this->webhookLogMapper->findAll(null, null); + $total = count(array_filter($allLogs, function ($log) use ($successBool) { + return $log->getSuccess() === $successBool; + })); + } + } + + return new JSONResponse( + data: [ + 'results' => $logs, + 'total' => $total, + ], + statusCode: 200 + ); + } catch (\Exception $e) { + $this->logger->error( + message: 'Error retrieving webhook logs: '.$e->getMessage(), + context: [ + 'trace' => $e->getTraceAsString(), + ] + ); + + return new JSONResponse( + data: [ + 'error' => 'Failed to retrieve webhook logs: '.$e->getMessage(), + ], + statusCode: 500 + ); + }//end try + + }//end allLogs() + + + /** + * Retry a failed webhook delivery + * + * @param int $logId Log entry ID + * + * @return JSONResponse + * + * @NoAdminRequired + * @NoCSRFRequired + */ + #[NoAdminRequired] + #[NoCSRFRequired] + public function retry(int $logId): JSONResponse + { + try { + // Get the log entry. + $log = $this->webhookLogMapper->find($logId); + + // Only allow retry for failed webhooks. + if ($log->getSuccess() === true) { + return new JSONResponse( + data: [ + 'error' => 'Cannot retry a successful webhook delivery', + ], + statusCode: 400 + ); + } + + // Get the webhook. + $webhook = $this->webhookMapper->find($log->getWebhookId()); + + // Extract payload from request body if available, otherwise use stored payload. + $payload = []; + if ($log->getRequestBody() !== null) { + $decoded = json_decode($log->getRequestBody(), true); + if ($decoded !== null) { + $payload = $decoded; + } + } elseif ($log->getPayload() !== null) { + $payload = $log->getPayloadArray(); + } + + // If no payload found, return error. + if (empty($payload)) { + return new JSONResponse( + data: [ + 'error' => 'No payload available for retry', + ], + statusCode: 400 + ); + } + + // Extract original event data from payload if available. + $eventName = $log->getEventClass(); + $originalPayload = $payload['data'] ?? $payload; + + // Retry the webhook delivery. + $success = $this->webhookService->deliverWebhook( + webhook: $webhook, + eventName: $eventName, + payload: $originalPayload, + attempt: $log->getAttempt() + 1 + ); + + if ($success === true) { + return new JSONResponse( + data: [ + 'success' => true, + 'message' => 'Webhook retry delivered successfully', + ] + ); + } else { + // Get the latest log entry to retrieve error details. + $latestLogs = $this->webhookLogMapper->findByWebhook($webhook->getId(), 1, 0); + $errorMessage = 'Webhook retry delivery failed'; + $errorDetails = null; + + if (!empty($latestLogs)) { + $latestLog = $latestLogs[0]; + if ($latestLog->getErrorMessage() !== null) { + $errorMessage = $latestLog->getErrorMessage(); + } + if ($latestLog->getStatusCode() !== null) { + $errorDetails = [ + 'status_code' => $latestLog->getStatusCode(), + 'response_body' => $latestLog->getResponseBody(), + ]; + } + } + + $responseData = [ + 'success' => false, + 'message' => $errorMessage, + ]; + + if ($errorDetails !== null) { + $responseData['error_details'] = $errorDetails; + } + + return new JSONResponse( + data: $responseData, + statusCode: 500 + ); + } + } catch (DoesNotExistException $e) { + $this->logger->error( + message: 'Webhook log not found for retry: '.$e->getMessage(), + context: [ + 'log_id' => $logId, + 'trace' => $e->getTraceAsString(), + ] + ); + + return new JSONResponse( + data: [ + 'error' => 'Webhook log not found', + ], + statusCode: 404 + ); + } catch (\Exception $e) { + $this->logger->error( + message: 'Error retrying webhook: '.$e->getMessage(), + context: [ + 'log_id' => $logId, + 'trace' => $e->getTraceAsString(), + ] + ); + + return new JSONResponse( + data: [ + 'error' => 'Failed to retry webhook', + ], + statusCode: 500 + ); + }//end try + + }//end retry() + + }//end class diff --git a/lib/Cron/ConfigurationCheckJob.php b/lib/Cron/ConfigurationCheckJob.php index 44cfba153..5cf581d34 100644 --- a/lib/Cron/ConfigurationCheckJob.php +++ b/lib/Cron/ConfigurationCheckJob.php @@ -100,7 +100,7 @@ public function __construct( $this->logger = $logger; // Set interval based on app configuration (default 3600 seconds = 1 hour). - $interval = (int) $this->appConfig->getValueString(appId: 'openregister', key: 'configuration_check_interval', default: '3600'); + $interval = (int) $this->appConfig->getValueString('openregister', 'configuration_check_interval', '3600'); // If interval is 0, disable the job by setting a very long interval. if ($interval === 0) { @@ -130,7 +130,7 @@ protected function run($argument): void $this->logger->info('Starting configuration check job'); // Check if the job is disabled. - $interval = (int) $this->appConfig->getValueString(appId: 'openregister', key: 'configuration_check_interval', default: '3600'); + $interval = (int) $this->appConfig->getValueString('openregister', 'configuration_check_interval', '3600'); if ($interval === 0) { $this->logger->info('Configuration check job is disabled, skipping'); return; diff --git a/lib/Cron/SyncConfigurationsJob.php b/lib/Cron/SyncConfigurationsJob.php index 95db0dd45..1140010ef 100644 --- a/lib/Cron/SyncConfigurationsJob.php +++ b/lib/Cron/SyncConfigurationsJob.php @@ -166,8 +166,8 @@ protected function run($argument): void $this->configurationMapper->updateSyncStatus( id: $configuration->getId(), status: 'failed', - syncedAt: new DateTime(), - errorMessage: $e->getMessage() + syncDate: new DateTime(), + message: $e->getMessage() ); } catch (Exception $statusError) { $this->logger->error("Failed to update sync status: ".$statusError->getMessage()); @@ -291,7 +291,7 @@ private function syncFromGitHub(Configuration $configuration): void $this->configurationMapper->updateSyncStatus( id: $configuration->getId(), status: 'success', - syncedAt: new DateTime() + syncDate: new DateTime() ); }//end syncFromGitHub() @@ -347,7 +347,7 @@ private function syncFromGitLab(Configuration $configuration): void $this->configurationMapper->updateSyncStatus( id: $configuration->getId(), status: 'success', - syncedAt: new DateTime() + syncDate: new DateTime() ); }//end syncFromGitLab() @@ -394,7 +394,7 @@ private function syncFromUrl(Configuration $configuration): void $this->configurationMapper->updateSyncStatus( id: $configuration->getId(), status: 'success', - syncedAt: new DateTime() + syncDate: new DateTime() ); }//end syncFromUrl() @@ -432,7 +432,7 @@ private function syncFromLocal(Configuration $configuration): void $this->configurationMapper->updateSyncStatus( id: $configuration->getId(), status: 'success', - syncedAt: new DateTime() + syncDate: new DateTime() ); }//end syncFromLocal() diff --git a/lib/Db/ObjectEntityMapper.php b/lib/Db/ObjectEntityMapper.php index 533f1443e..02f86743b 100644 --- a/lib/Db/ObjectEntityMapper.php +++ b/lib/Db/ObjectEntityMapper.php @@ -2510,8 +2510,8 @@ public function updateFromArray(int $id, array $object): ObjectEntity $newObject->hydrate($object); - // Prepare the object before updating. - return $this->update($this->prepareEntity($newObject)); + // Update the object. + return $this->update($newObject); }//end updateFromArray() diff --git a/lib/Db/OrganisationMapper.php b/lib/Db/OrganisationMapper.php index 9ade160aa..6eb621005 100644 --- a/lib/Db/OrganisationMapper.php +++ b/lib/Db/OrganisationMapper.php @@ -909,7 +909,7 @@ public function validateParentAssignment(string $organisationUuid, ?string $newP // Calculate maximum depth after assignment. $maxDepthAbove = count($parentChain) + 1; // Parent chain + new parent. - $maxDepthBelow = $this->getMaxDepthInChain(chain: $childrenChain, rootUuid: $organisationUuid); + $maxDepthBelow = $this->getMaxDepthInChain($childrenChain, $organisationUuid); $totalDepth = $maxDepthAbove + $maxDepthBelow; if ($totalDepth > 10) { @@ -965,7 +965,7 @@ private function getMaxDepthInChain(array $childrenUuids, string $rootUuid): int // Calculate depth for each child. $maxDepth = 0; foreach ($childrenUuids as $childUuid) { - $depth = $this->calculateDepthFromRoot(childUuid: $childUuid, rootUuid: $rootUuid, parentMap: $parentMap); + $depth = $this->calculateDepthFromRoot($childUuid, $rootUuid, $parentMap); $maxDepth = max($maxDepth, $depth); } diff --git a/lib/Db/WebhookLog.php b/lib/Db/WebhookLog.php index 3d50f4b95..0d96945de 100644 --- a/lib/Db/WebhookLog.php +++ b/lib/Db/WebhookLog.php @@ -44,6 +44,8 @@ * @method void setSuccess(bool $success) * @method int|null getStatusCode() * @method void setStatusCode(?int $statusCode) + * @method string|null getRequestBody() + * @method void setRequestBody(?string $requestBody) * @method string|null getResponseBody() * @method void setResponseBody(?string $responseBody) * @method string|null getErrorMessage() @@ -63,7 +65,7 @@ class WebhookLog extends Entity implements JsonSerializable * * @var integer */ - protected int $webhookId; + protected int $webhookId = 0; /** * Event class name @@ -107,6 +109,13 @@ class WebhookLog extends Entity implements JsonSerializable */ protected ?int $statusCode = null; + /** + * Request body (stored only on failure) + * + * @var string|null + */ + protected ?string $requestBody = null; + /** * Response body * @@ -157,12 +166,16 @@ public function __construct() $this->addType('method', 'string'); $this->addType('success', 'boolean'); $this->addType('statusCode', 'integer'); + $this->addType('requestBody', 'string'); $this->addType('responseBody', 'string'); $this->addType('errorMessage', 'string'); $this->addType('attempt', 'integer'); $this->addType('nextRetryAt', 'datetime'); $this->addType('created', 'datetime'); + // Initialize created timestamp. + $this->created = new DateTime(); + }//end __construct() @@ -217,6 +230,7 @@ public function jsonSerialize(): array 'method' => $this->method, 'success' => $this->success, 'statusCode' => $this->statusCode, + 'requestBody' => $this->requestBody, 'responseBody' => $this->responseBody, 'errorMessage' => $this->errorMessage, 'attempt' => $this->attempt, diff --git a/lib/Db/WebhookLogMapper.php b/lib/Db/WebhookLogMapper.php index bcf4b7c56..fbe9922ec 100644 --- a/lib/Db/WebhookLogMapper.php +++ b/lib/Db/WebhookLogMapper.php @@ -60,6 +60,29 @@ public function __construct(IDBConnection $db) }//end __construct() + /** + * Find a webhook log by ID + * + * @param int $id Log entry ID + * + * @return WebhookLog + * + * @throws \OCP\AppFramework\Db\DoesNotExistException + * @throws \OCP\AppFramework\Db\MultipleObjectsReturnedException + */ + public function find(int $id): WebhookLog + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where($qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT))); + + return $this->findEntity($qb); + + }//end find() + + /** * Find logs for a specific webhook * @@ -91,6 +114,35 @@ public function findByWebhook(int $webhookId, ?int $limit = null, ?int $offset = }//end findByWebhook() + /** + * Find all webhook logs + * + * @param int|null $limit Limit results + * @param int|null $offset Offset results + * + * @return WebhookLog[] + */ + public function findAll(?int $limit = null, ?int $offset = null): array + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->orderBy('created', 'DESC'); + + if ($limit !== null) { + $qb->setMaxResults($limit); + } + + if ($offset !== null) { + $qb->setFirstResult($offset); + } + + return $this->findEntities($qb); + + }//end findAll() + + /** * Find failed logs that need retry * @@ -126,9 +178,8 @@ public function findFailedForRetry(DateTime $before): array public function insert(Entity $entity): Entity { if ($entity instanceof WebhookLog) { - if ($entity->getCreated() === null) { - $entity->setCreated(new DateTime()); - } + // Always set created timestamp to ensure it's properly marked for insertion. + $entity->setCreated(new DateTime()); } return parent::insert($entity); diff --git a/lib/Listener/WebhookEventListener.php b/lib/Listener/WebhookEventListener.php index aab441e15..fdb3f60a9 100644 --- a/lib/Listener/WebhookEventListener.php +++ b/lib/Listener/WebhookEventListener.php @@ -130,7 +130,7 @@ public function handle(Event $event): void ); // Dispatch to webhook service. - $this->webhookService->dispatchEvent(event: $event, eventClass: $eventClass, payload: $payload); + $this->webhookService->dispatchEvent($event, $eventClass, $payload); }//end handle() diff --git a/lib/Migration/Version1Date20251127000000.php b/lib/Migration/Version1Date20251127000000.php new file mode 100644 index 000000000..1f622678d --- /dev/null +++ b/lib/Migration/Version1Date20251127000000.php @@ -0,0 +1,83 @@ + + * @copyright 2024 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * @version GIT: + * + * @link https://www.OpenRegister.app + */ + +declare(strict_types=1); + +namespace OCA\OpenRegister\Migration; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +/** + * Add request_body column to webhook_logs table + * + * @category Migration + * @package OCA\OpenRegister\Migration + * @author Conduction Development Team + * @copyright 2024 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version 1.0.0 + * @link https://www.OpenRegister.app + */ +class Version1Date20251127000000 extends SimpleMigrationStep +{ + + + /** + * Change database schema + * + * @param IOutput $output Output interface + * @param Closure $schemaClosure Schema closure + * @param array $options Options + * + * @return ISchemaWrapper|null + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper + { + /* + * @var ISchemaWrapper $schema + */ + $schema = $schemaClosure(); + + // Add request_body column to webhook_logs table if it exists. + if ($schema->hasTable('openregister_webhook_logs') === true) { + $table = $schema->getTable('openregister_webhook_logs'); + + // Add request_body column if it doesn't exist. + if ($table->hasColumn('request_body') === false) { + $table->addColumn( + 'request_body', + Types::TEXT, + [ + 'notnull' => false, + ] + ); + $output->info('Added request_body column to openregister_webhook_logs table'); + } + } + + return $schema; + + }//end changeSchema() + + +}//end class + diff --git a/lib/Service/FileService.php b/lib/Service/FileService.php index 9491a6c36..4d5b6e4de 100644 --- a/lib/Service/FileService.php +++ b/lib/Service/FileService.php @@ -440,10 +440,17 @@ private function createRegisterFolderById(Register $register, ?IUser $currentUse // Check if folder ID is already set and valid (not legacy string). if ($folderProperty !== null && $folderProperty !== '' && is_string($folderProperty) === false) { try { - $existingFolder = $this->getNodeById((int) $folderProperty); - if ($existingFolder !== null && $existingFolder instanceof Folder) { - $this->logger->info(message: "Register folder already exists with ID: " . $folderProperty); - return $existingFolder; + // Type assertion: after checking it's not a string, it should be numeric (int or float) + /** @var int|float $folderProperty */ + if (is_numeric($folderProperty)) { + $folderId = (int) $folderProperty; + $existingFolder = $this->getNodeById($folderId); + if ($existingFolder !== null && $existingFolder instanceof Folder) { + $this->logger->info(message: "Register folder already exists with ID: " . $folderProperty); + return $existingFolder; + } + } else { + throw new Exception('Invalid folder ID type'); } } catch (Exception $e) { $this->logger->warning(message: "Stored folder ID invalid, creating new folder: " . $e->getMessage()); @@ -500,10 +507,17 @@ private function createObjectFolderById(ObjectEntity|string $objectEntity, ?IUse // Check if folder ID is already set and valid (not legacy string). if ($folderProperty !== null && $folderProperty !== '' && is_string($folderProperty) === false) { try { - $existingFolder = $this->getNodeById((int) $folderProperty); - if ($existingFolder !== null && $existingFolder instanceof Folder) { - $this->logger->info(message: "Object folder already exists with ID: " . $folderProperty); - return $existingFolder; + // Type assertion: after checking it's not a string, it should be numeric (int or float) + /** @var int|float $folderProperty */ + if (is_numeric($folderProperty)) { + $folderId = (int) $folderProperty; + $existingFolder = $this->getNodeById($folderId); + if ($existingFolder !== null && $existingFolder instanceof Folder) { + $this->logger->info(message: "Object folder already exists with ID: " . $folderProperty); + return $existingFolder; + } + } else { + throw new Exception('Invalid folder ID type'); } } catch (Exception $e) { $this->logger->warning(message: "Stored folder ID invalid, creating new folder: " . $e->getMessage()); @@ -680,7 +694,14 @@ private function getRegisterFolderById(Register $register): ?Folder } // Try to get folder by ID. - $folder = $this->getNodeById((int) $folderProperty); + /** @var int|float $folderProperty */ + if (is_numeric($folderProperty)) { + $folderId = (int) $folderProperty; + $folder = $this->getNodeById($folderId); + } else { + $this->logger->warning(message: "Invalid folder ID type for register {$register->getId()}, creating new folder"); + return $this->createRegisterFolderById(register: $register); + } if ($folder instanceof Folder) { return $folder; @@ -1656,9 +1677,9 @@ private function shareFileWithUser(File $file, string $userId, int $permissions try { // Check if a share already exists with this user. $existingShares = $this->shareManager->getSharesBy( - sharedBy: $this->getUser()->getUID(), + userId: $this->getUser()->getUID(), shareType: \OCP\Share\IShare::TYPE_USER, - node: $file + path: $file ); foreach ($existingShares as $share) { diff --git a/lib/Service/GuzzleSolrService.php b/lib/Service/GuzzleSolrService.php index 0cf707b83..320f7095b 100644 --- a/lib/Service/GuzzleSolrService.php +++ b/lib/Service/GuzzleSolrService.php @@ -919,6 +919,17 @@ public function getActiveCollectionName(): ?string }//end getActiveCollectionName() + /** + * Get tenant collection name (alias for getActiveCollectionName) + * + * @return string|null The tenant collection name or null if no collection exists + */ + public function getTenantCollectionName(): ?string + { + return $this->getActiveCollectionName(); + }//end getTenantCollectionName() + + /** * Create SOLR collection * diff --git a/lib/Service/MagicMapper.php b/lib/Service/MagicMapper.php index 031e9625d..efe139060 100644 --- a/lib/Service/MagicMapper.php +++ b/lib/Service/MagicMapper.php @@ -1642,7 +1642,7 @@ private function convertRowToObjectEntity(array $row, Register $register, Schema // Handle datetime fields. if (in_array($metadataField, ['created', 'updated', 'published', 'depublished', 'expires']) && $value) { - $value = new \DateTime(datetime: $value); + $value = new \DateTime($value); } // Handle JSON fields. diff --git a/lib/Service/MagicMapperHandlers/MagicSearchHandler.php b/lib/Service/MagicMapperHandlers/MagicSearchHandler.php index a7e76cb2d..c5ef5087e 100644 --- a/lib/Service/MagicMapperHandlers/MagicSearchHandler.php +++ b/lib/Service/MagicMapperHandlers/MagicSearchHandler.php @@ -513,7 +513,7 @@ public function countObjects(array $query, Register $register, Schema $schema, s $countQuery = $query; $countQuery['_count'] = true; - return $this->searchObjects(query: $countQuery, activeOrganisationUuid: $register, rbac: $schema, multi: $tableName); + return $this->searchObjects($countQuery, $register, $schema, $tableName); }//end countObjects() diff --git a/lib/Service/NamedEntityRecognitionService.php b/lib/Service/NamedEntityRecognitionService.php index 8e88e881e..cbf6ad40f 100644 --- a/lib/Service/NamedEntityRecognitionService.php +++ b/lib/Service/NamedEntityRecognitionService.php @@ -27,6 +27,7 @@ use OCA\OpenRegister\Db\GdprEntity; use OCA\OpenRegister\Db\GdprEntityMapper; use OCP\AppFramework\Db\DoesNotExistException; +use OCP\IDBConnection; use Psr\Log\LoggerInterface; use Ramsey\Uuid\Uuid; @@ -83,7 +84,8 @@ class NamedEntityRecognitionService * @param EntityRelationMapper $entityRelationMapper Entity relation mapper. * @param ChunkMapper $chunkMapper Chunk mapper. * @param SettingsService $settingsService Settings service. - * @param LoggerInterface $logger Logger. + * @param IDBConnection $db Database connection. + * @param LoggerInterface $logger Logger. */ public function __construct( private readonly GdprEntityMapper $entityMapper, @@ -92,6 +94,7 @@ public function __construct( private readonly ChunkMapper $chunkMapper, /** @psalm-suppress UnusedProperty - Property kept for future use */ private readonly SettingsService $settingsService, + private readonly IDBConnection $db, private readonly LoggerInterface $logger ) { @@ -466,20 +469,22 @@ private function findOrCreateEntity(string $type, string $value, string $categor { // Try to find existing entity by value and type. try { - $qb = $this->entityMapper->getQueryBuilder(); + $qb = $this->db->getQueryBuilder(); $qb->select('*') ->from('openregister_entities') ->where($qb->expr()->eq('type', $qb->createNamedParameter($type))) ->andWhere($qb->expr()->eq('value', $qb->createNamedParameter($value))) ->setMaxResults(1); - $existing = $this->entityMapper->findEntity($qb); - - // Update timestamp. - $existing->setUpdatedAt(new DateTime()); - $this->entityMapper->update($existing); - - return $existing; + $existingEntities = $this->entityMapper->findEntities($qb); + if (empty($existingEntities) === false) { + $existing = $existingEntities[0]; + // Update timestamp. + $existing->setUpdatedAt(new DateTime()); + $this->entityMapper->update($existing); + return $existing; + } + throw new DoesNotExistException('Entity not found'); } catch (DoesNotExistException $e) { // Entity doesn't exist, create new one. $entity = new GdprEntity(); diff --git a/lib/Service/ObjectCacheService.php b/lib/Service/ObjectCacheService.php index d50f23f12..3f1913c46 100644 --- a/lib/Service/ObjectCacheService.php +++ b/lib/Service/ObjectCacheService.php @@ -1069,8 +1069,16 @@ public function invalidateForObjectChange( // **SCHEMA-WIDE INVALIDATION**: Clear ALL search caches for this schema. // This ensures colleagues see each other's changes immediately. - $schemaIdInt = ($schemaId !== null) ? (is_string($schemaId) ? (int) $schemaId : (int) $schemaId) : null; - $registerIdInt = ($registerId !== null) ? (is_string($registerId) ? (int) $registerId : (int) $registerId) : null; + $schemaIdInt = null; + if ($schemaId !== null) { + /** @var int|string $schemaId */ + $schemaIdInt = is_string($schemaId) ? (int) $schemaId : (int) $schemaId; + } + $registerIdInt = null; + if ($registerId !== null) { + /** @var int|string $registerId */ + $registerIdInt = is_string($registerId) ? (int) $registerId : (int) $registerId; + } $this->clearSchemaRelatedCaches(schemaId: $schemaIdInt, registerId: $registerIdInt, operation: $operation); $executionTime = round((microtime(true) - $startTime) * 1000, 2); diff --git a/lib/Service/ObjectService.php b/lib/Service/ObjectService.php index c1cd393c7..606b9e756 100644 --- a/lib/Service/ObjectService.php +++ b/lib/Service/ObjectService.php @@ -1364,7 +1364,7 @@ public function deleteObject(string $uuid, bool $rbac=true, bool $multi=true): b register: $this->currentRegister, schema: $this->currentSchema, uuid: $uuid, - folderId: null, + originalObjectId: null, rbac: $rbac, multi: $multi ); @@ -2222,7 +2222,7 @@ public function searchObjects( $this->logger->debug(message: 'Ultra-fast rendering completed', context: [ 'renderTime' => $simpleRenderTime . 'ms', 'objectCount' => count($objects), - 'avgPerObject' => $this->calculateAvgPerObject(objectCount: count($objects), renderTime: $simpleRenderTime), + 'avgPerObject' => $this->calculateAvgPerObject(count($objects), $simpleRenderTime), 'pathType' => 'ultra-fast-minimal' ]); @@ -2622,7 +2622,7 @@ public function getFacetsForObjects(array $query=[]): array public function getFacetableFields(array $baseQuery=[], int $sampleSize=100): array { // **ARCHITECTURAL IMPROVEMENT**: Delegate to dedicated FacetService. - return $this->facetService->getFacetableFields(baseQuery: $baseQuery, sampleSize: $sampleSize); + return $this->facetService->getFacetableFields($baseQuery, $sampleSize); }//end getFacetableFields() @@ -4822,7 +4822,7 @@ private function transferObjectFiles(ObjectEntity $sourceObject, ObjectEntity $t ); // Delete original file from source. - $this->fileService->deleteFile(file: $file, objectEntity: $sourceObject); + $this->fileService->deleteFile(file: $file, object: $sourceObject); $result['files'][] = [ 'name' => $fileName, @@ -5521,13 +5521,13 @@ private function logSearchTrail(array $query, int $resultCount, int $totalResult // Only create search trail if search trails are enabled. if ($this->isSearchTrailsEnabled() === true) { // Create the search trail entry using the service with actual execution time. - $this->searchTrailService->createSearchTrail( - query: $query, - resultCount: $resultCount, - totalResults: $totalResults, - executionTime: $executionTime, - executionType: $executionType - ); + $this->searchTrailService->createSearchTrail( + query: $query, + resultCount: $resultCount, + totalResults: $totalResults, + responseTime: $executionTime, + executionType: $executionType + ); } } catch (\Exception $e) { // Log the error but don't fail the request. diff --git a/lib/Service/SettingsService.php b/lib/Service/SettingsService.php index 7685f54ad..366f2ba3b 100644 --- a/lib/Service/SettingsService.php +++ b/lib/Service/SettingsService.php @@ -39,6 +39,7 @@ use OCA\OpenRegister\Service\SchemaCacheService; use OCA\OpenRegister\Service\SchemaFacetCacheService; use OCP\ICacheFactory; +use Psr\Log\LoggerInterface; /** * Service for handling settings-related operations. @@ -139,6 +140,7 @@ class SettingsService * @param SchemaCacheService $schemaCacheService Schema cache service for cache management. * @param SchemaFacetCacheService $schemaFacetCacheService Schema facet cache service for cache management. * @param ICacheFactory $cacheFactory Cache factory for distributed cache access. + * @param LoggerInterface $logger Logger for error and warning logging. */ public function __construct( private readonly IAppConfig $config, @@ -154,7 +156,8 @@ public function __construct( private readonly ObjectEntityMapper $objectEntityMapper, private readonly SchemaCacheService $schemaCacheService, private readonly SchemaFacetCacheService $schemaFacetCacheService, - private readonly ICacheFactory $cacheFactory + private readonly ICacheFactory $cacheFactory, + private readonly LoggerInterface $logger ) { // Indulge in setting the application name for identification and configuration purposes. $this->appName = 'openregister'; @@ -1301,7 +1304,7 @@ private function clearFacetCache(?string $userId = null): array return [ 'service' => 'facet', - 'cleared' => $beforeStats['entries'] - $afterStats['entries'], + 'cleared' => ($beforeStats['total_entries'] ?? 0) - ($afterStats['total_entries'] ?? 0), 'before' => $beforeStats, 'after' => $afterStats, 'success' => true, @@ -1319,9 +1322,11 @@ private function clearFacetCache(?string $userId = null): array /** * Clear distributed cache * - * @param string|null $userId Specific user ID + * @param string|null $userId Specific user ID (unused, kept for API compatibility) * * @return array Clear operation results + * + * @psalm-suppress UnusedParam */ private function clearDistributedCache(?string $userId = null): array { @@ -1579,34 +1584,33 @@ private function testSolrConnectivity(array $solrSettings): array $data = json_decode($response, true); // Validate admin response - be flexible about response format - if ($testType === 'admin_ping') { - // Check for successful response - different endpoints have different formats - $isValidResponse = false; - - if (isset($data['status']) && $data['status'] === 'OK') { - // Standard ping response - $isValidResponse = true; - } elseif (isset($data['responseHeader']['status']) && $data['responseHeader']['status'] === 0) { - // System info response - $isValidResponse = true; - } elseif (is_array($data) && !empty($data)) { - // Any valid JSON response indicates SOLR is responding - $isValidResponse = true; - } - - if (!$isValidResponse) { - return [ - 'success' => false, - 'message' => 'SOLR admin endpoint returned invalid response', - 'details' => [ - 'url' => $testUrl, - 'test_type' => $testType, - 'response' => $data, - 'response_time_ms' => round($responseTime, 2) - ] - ]; - } - + // Check for successful response - different endpoints have different formats + $isValidResponse = false; + + if (isset($data['status']) && $data['status'] === 'OK') { + // Standard ping response + $isValidResponse = true; + } elseif (isset($data['responseHeader']['status']) && $data['responseHeader']['status'] === 0) { + // System info response + $isValidResponse = true; + } elseif (is_array($data) && !empty($data)) { + // Any valid JSON response indicates SOLR is responding + $isValidResponse = true; + } + + if (!$isValidResponse) { + return [ + 'success' => false, + 'message' => 'SOLR admin endpoint returned invalid response', + 'details' => [ + 'url' => $testUrl, + 'test_type' => $testType, + 'response' => $data, + 'response_time_ms' => round($responseTime, 2) + ] + ]; + } + return [ 'success' => true, 'message' => 'SOLR server responding correctly', @@ -1620,32 +1624,6 @@ private function testSolrConnectivity(array $solrSettings): array 'working_endpoint' => str_replace($baseUrl, '', $testUrl) ] ]; - } else { - // For standalone admin ping test - if (!isset($data['status']) || $data['status'] !== 'OK') { - return [ - 'success' => false, - 'message' => 'SOLR admin ping failed', - 'details' => [ - 'url' => $testUrl, - 'test_type' => $testType, - 'response' => $data, - 'response_time_ms' => round($responseTime, 2) - ] - ]; - } - - return [ - 'success' => true, - 'message' => 'SOLR standalone server responding correctly', - 'details' => [ - 'url' => $testUrl, - 'test_type' => $testType, - 'response_time_ms' => round($responseTime, 2), - 'solr_version' => $data['lucene']['solr-spec-version'] ?? 'unknown' - ] - ]; - } } catch (Exception $e) { return [ @@ -2707,6 +2685,32 @@ public function getDefaultOrganisationUuid(): ?string } } + /** + * Get tenant ID from multitenancy settings + * + * @return string|null Tenant ID (default user tenant) or null if not set + */ + public function getTenantId(): ?string + { + try { + $multitenancySettings = $this->getMultitenancySettingsOnly(); + return $multitenancySettings['multitenancy']['defaultUserTenant'] ?? null; + } catch (Exception $e) { + $this->logger->warning('Failed to get tenant ID: '.$e->getMessage()); + return null; + } + } + + /** + * Get organisation ID (alias for getDefaultOrganisationUuid) + * + * @return string|null Organisation ID or null if not set + */ + public function getOrganisationId(): ?string + { + return $this->getDefaultOrganisationUuid(); + } + /** * Set default organisation UUID in settings * diff --git a/lib/Service/SolrFileService.php b/lib/Service/SolrFileService.php index 0c842fbda..c391ca239 100644 --- a/lib/Service/SolrFileService.php +++ b/lib/Service/SolrFileService.php @@ -704,7 +704,7 @@ private function jsonToText($data, string $prefix=''): string */ private function commandExists(string $command): bool { - // @psalm-suppress ForbiddenCode - shell_exec needed to check if command exists. + /** @psalm-suppress ForbiddenCode */ $result = shell_exec(sprintf('which %s 2>/dev/null', escapeshellarg($command))); return !empty($result); @@ -806,6 +806,28 @@ private function cleanText(string $text): string }//end cleanText() + /** + * Calculate average chunk size from an array of chunks. + * + * @param array $chunks Array of chunk strings. + * + * @return float Average chunk size in characters. + */ + private function calculateAvgChunkSize(array $chunks): float + { + if (count($chunks) === 0) { + return 0.0; + } + + $totalSize = 0; + foreach ($chunks as $chunk) { + $totalSize += strlen($chunk); + } + + return round($totalSize / count($chunks), 2); + }//end calculateAvgChunkSize() + + /** * Chunk text using fixed size with overlap * @@ -1267,20 +1289,18 @@ public function processExtractedFiles(?int $limit=null, array $options=[]): arra if ($result['success'] === true) { $stats['indexed']++; - // @psalm-suppress InvalidArrayOffset - $stats['total_chunks'] += $result['chunks_indexed']; + $stats['total_chunks'] += $result['indexed'] ?? 0; $this->logger->info( 'Successfully indexed file chunks', [ 'file_id' => $fileText->getFileId(), - 'chunks' => $result['chunks_indexed'], + 'chunks' => $result['indexed'] ?? 0, ] ); } else { $stats['failed']++; - // @psalm-suppress InvalidArrayOffset - $stats['errors'][$fileText->getFileId()] = $result['message'] ?? 'Unknown error'; + $stats['errors'][$fileText->getFileId()] = $result['error'] ?? ($result['message'] ?? 'Unknown error'); }//end if } catch (\Exception $e) { $stats['failed']++; @@ -1331,11 +1351,17 @@ public function processExtractedFile(int $fileId, array $options=[]): array throw new \Exception('fileCollection not configured in SOLR settings'); } + // Extract chunking options if provided + $chunkSize = $options['chunk_size'] ?? null; + $chunkOverlap = $options['chunk_overlap'] ?? null; + $this->logger->info( 'Processing single extracted file', [ - 'file_id' => $fileId, - 'collection' => $collection, + 'file_id' => $fileId, + 'collection' => $collection, + 'chunk_size' => $chunkSize, + 'chunk_overlap' => $chunkOverlap, ] ); @@ -1434,7 +1460,7 @@ public function getChunkingStats(): array 'total_extracted' => $extractionStats['totalFiles'] ?? 0, 'total_chunks_indexed' => $fileStats['document_count'] ?? 0, 'unique_files_indexed' => $fileStats['indexed_files'] ?? 0, - 'pending_indexing' => max(0, ($extractionStats['completed'] ?? 0) - ($fileStats['indexed_files'] ?? 0)), + 'pending_indexing' => max(0, ($extractionStats['totalFiles'] ?? 0) - ($fileStats['indexed_files'] ?? 0)), ]; }//end getChunkingStats() diff --git a/lib/Service/SolrObjectService.php b/lib/Service/SolrObjectService.php index b58a3eaf3..89ebdb42a 100644 --- a/lib/Service/SolrObjectService.php +++ b/lib/Service/SolrObjectService.php @@ -526,10 +526,12 @@ public function reindexObjects(int $maxObjects=0, int $batchSize=1000, array $sc 'collection' => $collection, 'maxObjects' => $maxObjects, 'batchSize' => $batchSize, + 'schemaIds' => $schemaIds, ] ); - // TODO: Move reindex logic to use collection parameter. + // Note: schemaIds parameter is logged but not yet used in reindexAll + // Future enhancement: filter reindexing by specific schema IDs return $this->guzzleSolrService->reindexAll(maxObjects: $maxObjects, batchSize: $batchSize); }//end reindexObjects() @@ -688,7 +690,6 @@ public function vectorizeObjects(array $objects, ?string $provider=null): array 'Starting batch object vectorization', [ 'total_objects' => count($objects), - // @psalm-suppress UndefinedMethod. 'provider' => $this->getProviderOrDefault($provider), ] ); @@ -815,4 +816,17 @@ public function vectorizeObjects(array $objects, ?string $provider=null): array }//end vectorizeObjects() + /** + * Get provider or return default value. + * + * @param string|null $provider Optional provider name. + * + * @return string Provider name or 'default' if not provided. + */ + private function getProviderOrDefault(?string $provider): string + { + return $provider ?? 'default'; + }//end getProviderOrDefault() + + }//end class diff --git a/lib/Service/SolrSchemaService.php b/lib/Service/SolrSchemaService.php index 2a4c861b0..c55a20655 100644 --- a/lib/Service/SolrSchemaService.php +++ b/lib/Service/SolrSchemaService.php @@ -2,7 +2,7 @@ declare(strict_types=1); -/* +/** * SolrSchemaService - Multi-Tenant, Multi-App Schema Mirroring * * This service handles automatic mirroring of OpenRegister schemas to SOLR @@ -12,16 +12,16 @@ * * **Multi-App Architecture**: * - OpenRegister fields: `or_naam_s`, `or_beschrijving_t` - * - Calendar fields: `cal_title_s`, `cal_start_dt` + * - Calendar fields: `cal_title_s`, `cal_start_dt` * - Other apps: `{app_prefix}_{field}_{type}` * - * @category Service - * @package OCA\OpenRegister\Service - * @author OpenRegister Team + * @category Service + * @package OCA\OpenRegister\Service + * @author OpenRegister Team * @copyright 2024 OpenRegister - * @license AGPL-3.0-or-later - * @version 1.0.0 - * @link https://github.com/OpenRegister/OpenRegister + * @license AGPL-3.0-or-later + * @version 1.0.0 + * @link https://github.com/OpenRegister/OpenRegister */ namespace OCA\OpenRegister\Service; @@ -36,7 +36,7 @@ * SOLR Schema Service for multi-tenant schema mirroring * * Automatically mirrors OpenRegister schemas to SOLR collections with: - * - Tenant-specific schema isolation + * - Tenant-specific schema isolation * - Organization-aware field mapping * - Dynamic field type detection * - Schema change synchronization @@ -66,18 +66,8 @@ class SolrSchemaService * @var array */ private const RESERVED_FIELDS = [ - 'id', - 'uuid', - 'self_tenant', - '_text_', - '_version_', - '_root_', - '_nest_path_', - '_embedding_', - '_embedding_model_', - '_embedding_dim_', - '_confidence_', - '_classification_', + 'id', 'uuid', 'self_tenant', '_text_', '_version_', '_root_', '_nest_path_', + '_embedding_', '_embedding_model_', '_embedding_dim_', '_confidence_', '_classification_' ]; /** @@ -95,219 +85,162 @@ class SolrSchemaService */ private const CORE_METADATA_FIELDS = [ // Primary identifier (required by SOLR) - 'id' => 'string', - + 'id' => 'string', + // Object metadata - 'self_object_id' => 'pint', - 'self_uuid' => 'string', - 'self_tenant' => 'string', - + 'self_object_id' => 'pint', + 'self_uuid' => 'string', + 'self_tenant' => 'string', + // Register metadata - 'self_register' => 'pint', - 'self_register_id' => 'pint', - 'self_register_uuid' => 'string', - 'self_register_slug' => 'string', - + 'self_register' => 'pint', + 'self_register_id' => 'pint', + 'self_register_uuid' => 'string', + 'self_register_slug' => 'string', + // Schema metadata - 'self_schema' => 'pint', - 'self_schema_id' => 'pint', - 'self_schema_uuid' => 'string', - 'self_schema_slug' => 'string', - + 'self_schema' => 'pint', + 'self_schema_id' => 'pint', + 'self_schema_uuid' => 'string', + 'self_schema_slug' => 'string', + // Other core fields - 'self_organisation' => 'string', - 'self_owner' => 'string', - 'self_application' => 'string', - 'self_name' => 'string', - 'self_description' => 'text_general', - 'self_summary' => 'text_general', - 'self_image' => 'string', - 'self_slug' => 'string', - 'self_uri' => 'string', - 'self_version' => 'string', - 'self_size' => 'plong', - 'self_folder' => 'string', - 'self_locked' => 'boolean', + 'self_organisation' => 'string', + 'self_owner' => 'string', + 'self_application' => 'string', + 'self_name' => 'string', + 'self_description' => 'text_general', + 'self_summary' => 'text_general', + 'self_image' => 'string', + 'self_slug' => 'string', + 'self_uri' => 'string', + 'self_version' => 'string', + 'self_size' => 'plong', + 'self_folder' => 'string', + 'self_locked' => 'boolean', 'self_schema_version' => 'string', - + // Sortable string variants (for ordering on text fields) // These are single-valued, non-tokenized copies used for sorting/faceting - 'self_name_s' => 'string', - 'self_description_s' => 'string', - 'self_summary_s' => 'string', - 'self_slug_s' => 'string', - + 'self_name_s' => 'string', + 'self_description_s' => 'string', + 'self_summary_s' => 'string', + 'self_slug_s' => 'string', + // Timestamps - 'self_created' => 'pdate', - 'self_updated' => 'pdate', - 'self_published' => 'pdate', - 'self_depublished' => 'pdate', - + 'self_created' => 'pdate', + 'self_updated' => 'pdate', + 'self_published' => 'pdate', + 'self_depublished' => 'pdate', + // Complex fields - 'self_object' => 'string', - // JSON storage - not indexed, only for reconstruction - 'self_relations' => 'string', - // Multi-valued UUIDs (multiValued=true) - 'self_files' => 'string', - // Multi-valued file references (multiValued=true) - 'self_authorization' => 'string', - // JSON storage - not indexed, only for reconstruction - 'self_deleted' => 'string', - // JSON storage - not indexed, only for reconstruction + 'self_object' => 'string', // JSON storage - not indexed, only for reconstruction + 'self_relations' => 'string', // Multi-valued UUIDs (multiValued=true) + 'self_files' => 'string', // Multi-valued file references (multiValued=true) + 'self_authorization' => 'string', // JSON storage - not indexed, only for reconstruction + 'self_deleted' => 'string', // JSON storage - not indexed, only for reconstruction + // AI/ML vector metadata fields // Note: Actual vector data is stored in oc_openregister_vectors table for efficiency // These fields track vectorization status for hybrid search coordination - 'vector_indexed' => 'boolean', - // Whether this object has vector embeddings - 'vector_model' => 'string', - // Model used for embeddings (e.g., "text-embedding-3-small") - 'vector_dimensions' => 'pint', - // Number of dimensions (e.g., 768, 1536) - 'vector_chunk_count' => 'pint', - // Number of chunks for this object - 'vector_updated' => 'pdate', - // When vectors were last generated - 'self_validation' => 'string', - // JSON storage - not indexed, only for reconstruction - 'self_groups' => 'string', - // JSON storage - not indexed, only for reconstruction + 'vector_indexed' => 'boolean', // Whether this object has vector embeddings + 'vector_model' => 'string', // Model used for embeddings (e.g., "text-embedding-3-small") + 'vector_dimensions' => 'pint', // Number of dimensions (e.g., 768, 1536) + 'vector_chunk_count' => 'pint', // Number of chunks for this object + 'vector_updated' => 'pdate', // When vectors were last generated + 'self_validation' => 'string', // JSON storage - not indexed, only for reconstruction + 'self_groups' => 'string', // JSON storage - not indexed, only for reconstruction + // SOLR system fields that need explicit definition - '_text_' => 'text_general', - // Catch-all full-text search field + '_text_' => 'text_general', // Catch-all full-text search field + // AI/ML fields for future semantic search and classification features - '_embedding_' => 'pfloat', - // Vector embeddings for semantic search (multiValued=true) - '_embedding_model_' => 'string', - // Model identifier (e.g., 'openai-ada-002', 'sentence-transformers') - '_embedding_dim_' => 'pint', - // Embedding dimension count for validation - '_confidence_' => 'pfloat', - // ML confidence scores (0.0-1.0) - '_classification_' => 'string', - // Auto-classification results (multiValued=true) + '_embedding_' => 'pfloat', // Vector embeddings for semantic search (multiValued=true) + '_embedding_model_' => 'string', // Model identifier (e.g., 'openai-ada-002', 'sentence-transformers') + '_embedding_dim_' => 'pint', // Embedding dimension count for validation + '_confidence_' => 'pfloat', // ML confidence scores (0.0-1.0) + '_classification_' => 'string' // Auto-classification results (multiValued=true) ]; /** * File metadata fields for FILE collection - * + * * These fields are used for storing and searching file chunks with metadata. * Supports full-text search, faceting, and vector semantic search. - * + * * @var array Field name => SOLR field type */ private const FILE_METADATA_FIELDS = [ // Primary identifier (required by SOLR) - 'id' => 'string', - + 'id' => 'string', + // Nextcloud file metadata - 'file_id' => 'plong', - // Nextcloud file ID from oc_filecache - 'file_path' => 'string', - // Full path in Nextcloud - 'file_name' => 'string', - // File name with extension - 'file_basename' => 'string', - // Name without extension (for faceting) - 'file_extension' => 'string', - // File extension (pdf, docx, txt) - 'file_mime_type' => 'string', - // MIME type (application/pdf, text/plain) - 'file_size' => 'plong', - // File size in bytes - 'file_owner' => 'string', - // Nextcloud user who owns the file - 'file_created' => 'pdate', - // File creation timestamp - 'file_modified' => 'pdate', - // File modification timestamp - 'file_storage' => 'pint', - // Storage ID from Nextcloud - 'file_parent' => 'plong', - // Parent folder ID - 'file_checksum' => 'string', - // File checksum/hash for deduplication + 'file_id' => 'plong', // Nextcloud file ID from oc_filecache + 'file_path' => 'string', // Full path in Nextcloud + 'file_name' => 'string', // File name with extension + 'file_basename' => 'string', // Name without extension (for faceting) + 'file_extension' => 'string', // File extension (pdf, docx, txt) + 'file_mime_type' => 'string', // MIME type (application/pdf, text/plain) + 'file_size' => 'plong', // File size in bytes + 'file_owner' => 'string', // Nextcloud user who owns the file + 'file_created' => 'pdate', // File creation timestamp + 'file_modified' => 'pdate', // File modification timestamp + 'file_storage' => 'pint', // Storage ID from Nextcloud + 'file_parent' => 'plong', // Parent folder ID + 'file_checksum' => 'string', // File checksum/hash for deduplication + // File classification and tags - 'file_labels' => 'string', - // User-defined labels (multiValued=true) - 'file_tags' => 'string', - // Auto-generated tags (multiValued=true) - 'file_categories' => 'string', - // Categories (multiValued=true) - 'file_language' => 'string', - // Detected language (en, nl, de, etc.) + 'file_labels' => 'string', // User-defined labels (multiValued=true) + 'file_tags' => 'string', // Auto-generated tags (multiValued=true) + 'file_categories' => 'string', // Categories (multiValued=true) + 'file_language' => 'string', // Detected language (en, nl, de, etc.) + // Chunking information - 'chunk_index' => 'pint', - // Chunk number (0-based) - 'chunk_total' => 'pint', - // Total number of chunks for this file - 'chunk_text' => 'text_general', - // The actual chunk text content - 'chunk_length' => 'pint', - // Length of this chunk in characters - 'chunk_start_offset' => 'plong', - // Start position in original file - 'chunk_end_offset' => 'plong', - // End position in original file - 'chunk_page_number' => 'pint', - // Page number (for PDFs, docs) + 'chunk_index' => 'pint', // Chunk number (0-based) + 'chunk_total' => 'pint', // Total number of chunks for this file + 'chunk_text' => 'text_general', // The actual chunk text content + 'chunk_length' => 'pint', // Length of this chunk in characters + 'chunk_start_offset' => 'plong', // Start position in original file + 'chunk_end_offset' => 'plong', // End position in original file + 'chunk_page_number' => 'pint', // Page number (for PDFs, docs) + // Full-text search fields - 'text_content' => 'text_general', - // Full extracted text for search - 'text_preview' => 'string', - // Short preview/summary - 'text_title' => 'text_general', - // Extracted title (from metadata or first heading) - 'text_author' => 'string', - // Extracted author from metadata - 'text_subject' => 'string', - // Document subject/topic + 'text_content' => 'text_general', // Full extracted text for search + 'text_preview' => 'string', // Short preview/summary + 'text_title' => 'text_general', // Extracted title (from metadata or first heading) + 'text_author' => 'string', // Extracted author from metadata + 'text_subject' => 'string', // Document subject/topic + // OCR and extraction metadata - 'ocr_performed' => 'boolean', - // Whether OCR was performed - 'ocr_confidence' => 'pfloat', - // OCR confidence score (0.0-1.0) - 'extraction_method' => 'string', - // Method used (text_extract, ocr, api) - 'extraction_date' => 'pdate', - // When text was extracted + 'ocr_performed' => 'boolean', // Whether OCR was performed + 'ocr_confidence' => 'pfloat', // OCR confidence score (0.0-1.0) + 'extraction_method' => 'string', // Method used (text_extract, ocr, api) + 'extraction_date' => 'pdate', // When text was extracted + // Vector embedding metadata - 'vector_indexed' => 'boolean', - // Whether this chunk has vector embeddings - 'vector_model' => 'string', - // Model used for embeddings - 'vector_dimensions' => 'pint', - // Number of dimensions - 'vector_updated' => 'pdate', - // When vectors were last generated + 'vector_indexed' => 'boolean', // Whether this chunk has vector embeddings + 'vector_model' => 'string', // Model used for embeddings + 'vector_dimensions' => 'pint', // Number of dimensions + 'vector_updated' => 'pdate', // When vectors were last generated + // Relationships and context - 'related_object_id' => 'string', - // Related object UUID (if attached to object) - 'related_object_type' => 'string', - // Object type/schema - 'shared_with' => 'string', - // Users/groups with access (multiValued=true) - 'access_level' => 'string', - // Access level (public, shared, private) + 'related_object_id' => 'string', // Related object UUID (if attached to object) + 'related_object_type' => 'string', // Object type/schema + 'shared_with' => 'string', // Users/groups with access (multiValued=true) + 'access_level' => 'string', // Access level (public, shared, private) + // Processing status - 'processing_status' => 'string', - // Status (pending, processed, failed, skipped) - 'processing_error' => 'string', - // Error message if processing failed - 'processing_date' => 'pdate', - // When processing completed + 'processing_status' => 'string', // Status (pending, processed, failed, skipped) + 'processing_error' => 'string', // Error message if processing failed + 'processing_date' => 'pdate', // When processing completed + // AI/ML fields - '_text_' => 'text_general', - // Catch-all full-text search field - '_embedding_' => 'knn_vector', - // Vector embeddings (dense vector for KNN search) - '_embedding_model_' => 'string', - // Model identifier - '_embedding_dim_' => 'pint', - // Embedding dimension count - '_confidence_' => 'pfloat', - // ML confidence scores - '_classification_' => 'string', - // Auto-classification results (multiValued=true) + '_text_' => 'text_general', // Catch-all full-text search field + '_embedding_' => 'knn_vector', // Vector embeddings (dense vector for KNN search) + '_embedding_model_' => 'string', // Model identifier + '_embedding_dim_' => 'pint', // Embedding dimension count + '_confidence_' => 'pfloat', // ML confidence scores + '_classification_' => 'string' // Auto-classification results (multiValued=true) ]; /** @@ -316,25 +249,16 @@ class SolrSchemaService * @var array */ private array $fieldTypeMappings = [ - 'string' => '_s', - // String, facetable - 'text' => '_t', - // Text, searchable - 'integer' => '_i', - // Integer - 'number' => '_f', - // Float - 'boolean' => '_b', - // Boolean - 'date' => '_dt', - // Date/DateTime - 'array' => '_ss', - // String array - 'object' => '_json', - // JSON object (if supported) + 'string' => '_s', // String, facetable + 'text' => '_t', // Text, searchable + 'integer' => '_i', // Integer + 'number' => '_f', // Float + 'boolean' => '_b', // Boolean + 'date' => '_dt', // Date/DateTime + 'array' => '_ss', // String array + 'object' => '_json', // JSON object (if supported) ]; - /** * Constructor * @@ -350,38 +274,36 @@ public function __construct( private readonly LoggerInterface $logger, private readonly IConfig $config ) { - - }//end __construct() - + } /** * Ensure knn_vector field type exists in Solr for dense vector search - * + * * This method adds the knn_vector field type to the Solr schema if it doesn't exist. * Required for Solr 9+ dense vector search functionality. - * - * @param string $collection Collection name to configure - * @param int $dimensions Vector dimensions (default: 4096 for mistral:7b) - * @param string $similarity Similarity function: 'cosine', 'dot_product', or 'euclidean' - * + * + * @param string $collection Collection name to configure + * @param int $dimensions Vector dimensions (default: 4096 for mistral:7b) + * @param string $similarity Similarity function: 'cosine', 'dot_product', or 'euclidean' + * * @return bool Success status */ public function ensureVectorFieldType( - string $collection, - int $dimensions=4096, - string $similarity='cosine' + string $collection, + int $dimensions = 4096, + string $similarity = 'cosine' ): bool { try { - $settings = $this->settingsService->getSettings(); - $solrUrl = $this->solrService->buildSolrBaseUrl(); + $settings = $this->settingsService->getSettings(); + $solrUrl = $this->solrService->buildSolrBaseUrl(); $schemaUrl = "{$solrUrl}/{$collection}/schema"; // Check if knn_vector type already exists $checkUrl = "{$schemaUrl}/fieldtypes/knn_vector"; - + $requestOptions = [ 'timeout' => 30, - 'headers' => ['Accept' => 'application/json'], + 'headers' => ['Accept' => 'application/json'] ]; // Add authentication @@ -393,77 +315,60 @@ public function ensureVectorFieldType( try { $response = $this->solrService->getHttpClient()->get($checkUrl, $requestOptions); - $data = json_decode((string) $response->getBody(), true); - + $data = json_decode((string)$response->getBody(), true); + if (isset($data['fieldType'])) { - $this->logger->info( - 'knn_vector field type already exists', - [ - 'collection' => $collection, - ] - ); - return true; - // Already exists + $this->logger->info('knn_vector field type already exists', [ + 'collection' => $collection + ]); + return true; // Already exists } } catch (\Exception $e) { // Field type doesn't exist, continue to create it - $this->logger->debug( - 'knn_vector field type not found, creating', - [ - 'collection' => $collection, - ] - ); - }//end try + $this->logger->debug('knn_vector field type not found, creating', [ + 'collection' => $collection + ]); + } // Add knn_vector field type $payload = [ 'add-field-type' => [ - 'name' => 'knn_vector', - 'class' => 'solr.DenseVectorField', - 'vectorDimension' => $dimensions, - 'similarityFunction' => $similarity, - ], + 'name' => 'knn_vector', + 'class' => 'solr.DenseVectorField', + 'vectorDimension' => $dimensions, + 'similarityFunction' => $similarity + ] ]; $requestOptions['body'] = json_encode($payload); $requestOptions['headers']['Content-Type'] = 'application/json'; - $response = $this->solrService->getHttpClient()->post($schemaUrl, $requestOptions); - $responseData = json_decode((string) $response->getBody(), true); + $response = $this->solrService->getHttpClient()->post($schemaUrl, $requestOptions); + $responseData = json_decode((string)$response->getBody(), true); if (($responseData['responseHeader']['status'] ?? -1) === 0) { - $this->logger->info( - '✅ knn_vector field type created successfully', - [ - 'collection' => $collection, - 'dimensions' => $dimensions, - 'similarity' => $similarity, - ] - ); + $this->logger->info('✅ knn_vector field type created successfully', [ + 'collection' => $collection, + 'dimensions' => $dimensions, + 'similarity' => $similarity + ]); return true; } - $this->logger->error( - 'Failed to create knn_vector field type', - [ - 'response' => $responseData, - ] - ); + $this->logger->error('Failed to create knn_vector field type', [ + 'response' => $responseData + ]); return false; + } catch (\Exception $e) { - $this->logger->error( - 'Exception creating knn_vector field type', - [ - 'error' => $e->getMessage(), - 'collection' => $collection, - ] - ); + $this->logger->error('Exception creating knn_vector field type', [ + 'error' => $e->getMessage(), + 'collection' => $collection + ]); return false; - }//end try - + } }//end ensureVectorFieldType() - /** * Mirror all OpenRegister schemas to SOLR for current tenant with intelligent conflict resolution * @@ -471,33 +376,30 @@ public function ensureVectorFieldType( * and chooses the most permissive type (string > text > float > integer > boolean). * This prevents errors like "versie" being integer in one schema and string in another. * - * @param bool $force Force recreation of existing fields + * @param bool $force Force recreation of existing fields * @return array Result with success status and statistics */ - public function mirrorSchemas(bool $force=false): array + public function mirrorSchemas(bool $force = false): array { $startTime = microtime(true); - $stats = [ - 'schemas_processed' => 0, - 'fields_created' => 0, - 'fields_updated' => 0, + $stats = [ + 'schemas_processed' => 0, + 'fields_created' => 0, + 'fields_updated' => 0, 'conflicts_resolved' => 0, - 'errors' => 0, + 'errors' => 0 ]; try { // Generate tenant information - $tenantId = $this->generateTenantId(); - $organisationId = null; - // For now, process all schemas regardless of organization - $this->logger->info( - '🔄 Starting intelligent schema mirroring with conflict resolution', - [ - 'app' => 'openregister', - 'tenant_id' => $tenantId, - 'organisation_id' => $organisationId, - ] - ); + $tenantId = $this->generateTenantId(); + $organisationId = null; // For now, process all schemas regardless of organization + + $this->logger->info('🔄 Starting intelligent schema mirroring with conflict resolution', [ + 'app' => 'openregister', + 'tenant_id' => $tenantId, + 'organisation_id' => $organisationId + ]); // Ensure tenant collection exists if (!$this->solrService->ensureTenantCollection()) { @@ -506,60 +408,53 @@ public function mirrorSchemas(bool $force=false): array // Get all schemas (process all schemas regardless of organization for conflict resolution) $schemas = $this->schemaMapper->findAll(); - + // STEP 1: Analyze all schemas to detect field conflicts and resolve them $resolvedFields = $this->analyzeAndResolveFieldConflicts($schemas); $stats['conflicts_resolved'] = $resolvedFields['conflicts_resolved']; - + // STEP 2: Ensure core metadata fields exist in SOLR schema $this->ensureCoreMetadataFields($force); $stats['core_fields_created'] = count(self::CORE_METADATA_FIELDS); - + // STEP 3: Apply resolved field definitions to SOLR if (!empty($resolvedFields['fields'])) { $this->applySolrFields($resolvedFields['fields'], $force); $stats['fields_created'] = count($resolvedFields['fields']); } - + $stats['schemas_processed'] = count($schemas); $executionTime = round((microtime(true) - $startTime) * 1000, 2); - - $this->logger->info( - '✅ Intelligent schema mirroring completed', - [ - 'app' => 'openregister', - 'stats' => $stats, - 'execution_time_ms' => $executionTime, - 'resolved_conflicts' => $resolvedFields['conflict_details'] ?? [], - ] - ); + + $this->logger->info('✅ Intelligent schema mirroring completed', [ + 'app' => 'openregister', + 'stats' => $stats, + 'execution_time_ms' => $executionTime, + 'resolved_conflicts' => $resolvedFields['conflict_details'] ?? [] + ]); return [ - 'success' => true, - 'stats' => $stats, - 'execution_time_ms' => $executionTime, - 'resolved_conflicts' => $resolvedFields['conflict_details'] ?? [], + 'success' => true, + 'stats' => $stats, + 'execution_time_ms' => $executionTime, + 'resolved_conflicts' => $resolvedFields['conflict_details'] ?? [] ]; + } catch (\Exception $e) { - $this->logger->error( - 'Schema mirroring failed', - [ - 'app' => 'openregister', - 'error' => $e->getMessage(), - 'stats' => $stats, - ] - ); + $this->logger->error('Schema mirroring failed', [ + 'app' => 'openregister', + 'error' => $e->getMessage(), + 'stats' => $stats + ]); return [ 'success' => false, - 'error' => $e->getMessage(), - 'stats' => $stats, + 'error' => $e->getMessage(), + 'stats' => $stats ]; - }//end try - - }//end mirrorSchemas() - + } + } /** * Analyze all schemas and resolve field type conflicts with intelligent type selection @@ -567,11 +462,11 @@ public function mirrorSchemas(bool $force=false): array * **CONFLICT RESOLUTION STRATEGY**: * When the same field exists in multiple schemas with different types, we choose the * most permissive type that can accommodate all values: - * + * * **Type Priority (Most → Least Permissive)**: * 1. `string` - Can store any value (numbers, text, booleans as strings) * 2. `text` - Can store any text content with full-text search - * 3. `float` - Can store integers and decimals + * 3. `float` - Can store integers and decimals * 4. `integer` - Can only store whole numbers * 5. `boolean` - Can only store true/false * @@ -580,30 +475,26 @@ public function mirrorSchemas(bool $force=false): array * - Schema 67: `versie` = "integer" (values: 123, 456) * - **Resolution**: `versie` = "string" (can store both text and numbers) * - * @param array $schemas Array of Schema entities to analyze + * @param array $schemas Array of Schema entities to analyze * @return array Resolved field definitions with conflict details */ private function analyzeAndResolveFieldConflicts(array $schemas): array { - $fieldDefinitions = []; - // [fieldName => [type => count, schemas => [schema_ids]]] - $resolvedFields = []; - $conflictDetails = []; + $fieldDefinitions = []; // [fieldName => [type => count, schemas => [schema_ids]]] + $resolvedFields = []; + $conflictDetails = []; $conflictsResolved = 0; - $this->logger->info( - '🔍 Analyzing field conflicts across schemas', - [ - 'total_schemas' => count($schemas), - ] - ); + $this->logger->info('🔍 Analyzing field conflicts across schemas', [ + 'total_schemas' => count($schemas) + ]); // STEP 1: Collect all field definitions from all schemas foreach ($schemas as $schema) { - $schemaId = $schema->getId(); - $schemaTitle = $schema->getTitle(); + $schemaId = $schema->getId(); + $schemaTitle = $schema->getTitle(); $schemaProperties = $schema->getProperties(); - + if (empty($schemaProperties)) { continue; } @@ -611,24 +502,20 @@ private function analyzeAndResolveFieldConflicts(array $schemas): array // $schemaProperties is already an array from getProperties() $properties = $schemaProperties; if (!is_array($properties)) { - $this->logger->warning( - 'Invalid schema properties', - [ - 'schema_id' => $schemaId, - 'schema_title' => $schemaTitle, - 'properties_type' => gettype($properties), - ] - ); + $this->logger->warning('Invalid schema properties', [ + 'schema_id' => $schemaId, + 'schema_title' => $schemaTitle, + 'properties_type' => gettype($properties) + ]); continue; } // Collect field definitions foreach ($properties as $fieldName => $fieldDefinition) { $fieldType = $fieldDefinition['type'] ?? 'string'; - + // Skip reserved fields and metadata fields - // Cast fieldName to string to handle numeric keys - $fieldNameStr = (string) $fieldName; + $fieldNameStr = $fieldName; if (in_array($fieldNameStr, self::RESERVED_FIELDS) || str_starts_with($fieldNameStr, 'self_')) { continue; } @@ -636,9 +523,9 @@ private function analyzeAndResolveFieldConflicts(array $schemas): array // Initialize field tracking if (!isset($fieldDefinitions[$fieldNameStr])) { $fieldDefinitions[$fieldNameStr] = [ - 'types' => [], - 'schemas' => [], - 'definitions' => [], + 'types' => [], + 'schemas' => [], + 'definitions' => [] ]; } @@ -646,151 +533,133 @@ private function analyzeAndResolveFieldConflicts(array $schemas): array if (!isset($fieldDefinitions[$fieldNameStr]['types'][$fieldType])) { $fieldDefinitions[$fieldNameStr]['types'][$fieldType] = 0; } - $fieldDefinitions[$fieldNameStr]['types'][$fieldType]++; - $fieldDefinitions[$fieldNameStr]['schemas'][] = ['id' => $schemaId, 'title' => $schemaTitle]; + $fieldDefinitions[$fieldNameStr]['schemas'][] = ['id' => $schemaId, 'title' => $schemaTitle]; $fieldDefinitions[$fieldNameStr]['definitions'][] = $fieldDefinition; - }//end foreach - }//end foreach + } + } // STEP 2: Resolve conflicts by choosing most permissive type foreach ($fieldDefinitions as $fieldName => $fieldInfo) { - // Cast fieldName to string to handle numeric keys - $fieldNameStr = (string) $fieldName; - + $fieldNameStr = $fieldName; + $types = array_keys($fieldInfo['types']); - + if (count($types) > 1) { // CONFLICT DETECTED - resolve with most permissive type - $resolvedType = $this->getMostPermissiveType($types); + $resolvedType = $this->getMostPermissiveType($types); $conflictDetails[] = [ - 'field' => $fieldNameStr, + 'field' => $fieldNameStr, 'conflicting_types' => $fieldInfo['types'], - 'resolved_type' => $resolvedType, - 'schemas' => $fieldInfo['schemas'], + 'resolved_type' => $resolvedType, + 'schemas' => $fieldInfo['schemas'] ]; $conflictsResolved++; - - $this->logger->info( - '🔧 Field conflict resolved', - [ - 'field' => $fieldNameStr, - 'conflicting_types' => $types, - 'resolved_type' => $resolvedType, - 'affected_schemas' => count($fieldInfo['schemas']), - ] - ); + + $this->logger->info('🔧 Field conflict resolved', [ + 'field' => $fieldNameStr, + 'conflicting_types' => $types, + 'resolved_type' => $resolvedType, + 'affected_schemas' => count($fieldInfo['schemas']) + ]); } else { // No conflict - use the single type $resolvedType = $types[0]; - }//end if + } // Create SOLR field definition with resolved type - $solrFieldName = $this->generateSolrFieldName($fieldNameStr, $fieldInfo['definitions'][0]); + $solrFieldName = $this->generateSolrFieldName($fieldNameStr); $solrFieldType = $this->determineSolrFieldType(['type' => $resolvedType] + $fieldInfo['definitions'][0]); - + if ($solrFieldName && $solrFieldType) { // Apply most permissive settings by checking ALL definitions // If ANY schema has facetable=true, the field should support faceting // If ANY schema is multi-valued, the field should be multi-valued - $isFacetable = false; + $isFacetable = false; $isMultiValued = false; - + foreach ($fieldInfo['definitions'] as $definition) { if (($definition['facetable'] ?? false) === true) { $isFacetable = true; } - if ($this->isMultiValued($definition)) { $isMultiValued = true; } } - + $resolvedFields[$solrFieldName] = [ - 'type' => $solrFieldType, - 'stored' => true, - 'indexed' => true, + 'type' => $solrFieldType, + 'stored' => true, + 'indexed' => true, 'multiValued' => $isMultiValued, - 'docValues' => $isFacetable, - // docValues enabled when ANY schema needs faceting - 'facetable' => $isFacetable, + 'docValues' => $isFacetable, // docValues enabled when ANY schema needs faceting + 'facetable' => $isFacetable ]; + + $this->logger->debug('Field definition resolved', [ + 'field' => $solrFieldName, + 'type' => $solrFieldType, + 'multiValued' => $isMultiValued, + 'facetable' => $isFacetable, + 'definitions_checked' => count($fieldInfo['definitions']) + ]); + } + } - $this->logger->debug( - 'Field definition resolved', - [ - 'field' => $solrFieldName, - 'type' => $solrFieldType, - 'multiValued' => $isMultiValued, - 'facetable' => $isFacetable, - 'definitions_checked' => count($fieldInfo['definitions']), - ] - ); - }//end if - }//end foreach - - $this->logger->info( - '✅ Field conflict analysis completed', - [ - 'total_fields' => count($fieldDefinitions), - 'conflicts_detected' => $conflictsResolved, - 'resolved_fields' => count($resolvedFields), - ] - ); + $this->logger->info('✅ Field conflict analysis completed', [ + 'total_fields' => count($fieldDefinitions), + 'conflicts_detected' => $conflictsResolved, + 'resolved_fields' => count($resolvedFields) + ]); return [ - 'fields' => $resolvedFields, + 'fields' => $resolvedFields, 'conflicts_resolved' => $conflictsResolved, - 'conflict_details' => $conflictDetails, + 'conflict_details' => $conflictDetails ]; - - }//end analyzeAndResolveFieldConflicts() - + } /** * Determine the most permissive type from a list of conflicting types * * **Type Permissiveness Hierarchy** (most permissive first): * 1. `string` - Universal container (can hold any value as text) - * 2. `text` - Text with full-text search capabilities + * 2. `text` - Text with full-text search capabilities * 3. `float`/`double`/`number` - Can hold integers and decimals * 4. `integer`/`int` - Can only hold whole numbers * 5. `boolean` - Can only hold true/false values * - * @param array $types List of conflicting field types + * @param array $types List of conflicting field types * @return string Most permissive type that can accommodate all values */ private function getMostPermissiveType(array $types): string { // Define type hierarchy from most to least permissive $typeHierarchy = [ - 'string' => 100, - 'text' => 90, - 'float' => 80, - 'double' => 80, - 'number' => 80, + 'string' => 100, + 'text' => 90, + 'float' => 80, + 'double' => 80, + 'number' => 80, 'integer' => 70, - 'int' => 70, + 'int' => 70, 'boolean' => 60, - 'bool' => 60, + 'bool' => 60 ]; - $maxPermissiveness = 0; - $mostPermissiveType = 'string'; - // Default fallback + $maxPermissiveness = 0; + $mostPermissiveType = 'string'; // Default fallback + foreach ($types as $type) { - $permissiveness = $typeHierarchy[strtolower($type)] ?? 50; - // Unknown types get low priority + $permissiveness = $typeHierarchy[strtolower($type)] ?? 50; // Unknown types get low priority if ($permissiveness > $maxPermissiveness) { - $maxPermissiveness = $permissiveness; + $maxPermissiveness = $permissiveness; $mostPermissiveType = $type; } } return $mostPermissiveType; - - }//end getMostPermissiveType() - + } /** * Generate tenant ID for multi-tenant SOLR collections @@ -801,92 +670,77 @@ private function getMostPermissiveType(array $types): string */ private function generateTenantId(): string { - $instanceId = $this->config->getSystemValue('instanceid', 'default'); + $instanceId = $this->config->getSystemValue('instanceid', 'default'); $overwriteHost = $this->config->getSystemValue('overwrite.cli.url', ''); - + if (!empty($overwriteHost)) { - return 'nc_'.hash('crc32', $overwriteHost); + return 'nc_' . hash('crc32', $overwriteHost); } - - return 'nc_'.substr($instanceId, 0, 8); - - }//end generateTenantId() - + + return 'nc_' . substr($instanceId, 0, 8); + } /** * Mirror a single OpenRegister schema to SOLR * - * @param \OCA\OpenRegister\Db\Schema $schema OpenRegister schema entity - * @param bool $force Force update existing fields + * @param \OCA\OpenRegister\Db\Schema $schema OpenRegister schema entity + * @param bool $force Force update existing fields * @return array Field mapping results */ - private function mirrorSingleSchema($schema, bool $force=false): array + private function mirrorSingleSchema($schema, bool $force = false): array { - $schemaProperties = $schema->getProperties(); - if (empty($schemaProperties)) { + $properties = $schema->getProperties(); + if (empty($properties) || !is_array($properties)) { return ['fields' => 0, 'message' => 'No properties to mirror']; } - $properties = json_decode($schemaProperties, true); - if (!is_array($properties)) { - return ['fields' => 0, 'message' => 'Invalid schema properties JSON']; - } - $fieldsCreated = 0; - $solrFields = []; + $solrFields = []; // Convert OpenRegister properties to SOLR fields foreach ($properties as $fieldName => $fieldDefinition) { - $solrFieldName = $this->generateSolrFieldName($fieldName, $fieldDefinition); + $solrFieldName = $this->generateSolrFieldName($fieldName); $solrFieldType = $this->determineSolrFieldType($fieldDefinition); - + if ($solrFieldName && $solrFieldType) { $isFacetable = $fieldDefinition['facetable'] ?? true; - + // **FILE TYPE HANDLING**: File fields should not be indexed to avoid size limits - $type = $fieldDefinition['type'] ?? 'string'; - $format = $fieldDefinition['format'] ?? ''; - $isFileField = ($type === 'file' || $format === 'file' || $format === 'binary' || + $type = $fieldDefinition['type'] ?? 'string'; + $format = $fieldDefinition['format'] ?? ''; + $isFileField = ($type === 'file' || $format === 'file' || $format === 'binary' || in_array($format, ['data-url', 'base64', 'image', 'document'])); - + $solrFields[$solrFieldName] = [ - 'type' => $solrFieldType, - 'stored' => true, - 'indexed' => !$isFileField, - // File fields are stored but not indexed + 'type' => $solrFieldType, + 'stored' => true, + 'indexed' => !$isFileField, // File fields are stored but not indexed 'multiValued' => $this->isMultiValued($fieldDefinition), - 'docValues' => $isFacetable && !$isFileField, - // File fields can't have docValues - 'facetable' => $isFacetable && !$isFileField, - // File fields can't be faceted + 'docValues' => $isFacetable && !$isFileField, // File fields can't have docValues + 'facetable' => $isFacetable && !$isFileField // File fields can't be faceted ]; $fieldsCreated++; - }//end if - }//end foreach + } + } // Apply fields to SOLR collection if (!empty($solrFields)) { $this->applySolrFields($solrFields, $force); } - $this->logger->debug( - 'Schema mirrored', - [ - 'app' => 'openregister', - 'schema_id' => $schema->getId(), - 'schema_title' => $schema->getTitle(), - 'fields_processed' => $fieldsCreated, - 'solr_fields' => array_keys($solrFields), - ] - ); + $this->logger->debug('Schema mirrored', [ + 'app' => 'openregister', + 'schema_id' => $schema->getId(), + 'schema_title' => $schema->getTitle(), + 'fields_processed' => $fieldsCreated, + 'solr_fields' => array_keys($solrFields) + ]); return [ - 'fields' => $fieldsCreated, - 'solr_fields' => $solrFields, + 'fields' => $fieldsCreated, + 'solr_fields' => $solrFields ]; - - }//end mirrorSingleSchema() - + } /** * Generate SOLR field name with consistent self_ prefix (no suffixes needed) @@ -896,11 +750,11 @@ private function mirrorSingleSchema($schema, bool $force=false): array * - Metadata fields: `self_` prefix (e.g., `self_name`, `self_description`) * - Reserved fields: `id`, `uuid`, `self_tenant` (no additional prefix) * - * @param string $fieldName OpenRegister field name - * @param array $fieldDefinition Field definition from schema + * @param string $fieldName OpenRegister field name + * @param array $fieldDefinition Field definition from schema * @return string SOLR field name with consistent naming */ - private function generateSolrFieldName(string $fieldName, array $fieldDefinition): string + private function generateSolrFieldName(string $fieldName): string { // Don't prefix reserved fields if (in_array($fieldName, self::RESERVED_FIELDS)) { @@ -909,12 +763,10 @@ private function generateSolrFieldName(string $fieldName, array $fieldDefinition // Clean field name (SOLR field names have restrictions) $cleanName = preg_replace('/[^a-zA-Z0-9_]/', '_', $fieldName); - + // Use direct field names for object data (no prefixes or suffixes needed with explicit schema) return $cleanName; - - }//end generateSolrFieldName() - + } /** * Determine SOLR field type from OpenRegister field definition @@ -930,60 +782,50 @@ private function generateSolrFieldName(string $fieldName, array $fieldDefinition * - `string`: Exact matching for IDs, codes, names, faceting * - `docValues`: Enables fast sorting/faceting but uses more storage * - * @param array $fieldDefinition OpenRegister field definition + * @param array $fieldDefinition OpenRegister field definition * @return string SOLR field type */ private function determineSolrFieldType(array $fieldDefinition): string { - $type = $fieldDefinition['type'] ?? 'string'; + $type = $fieldDefinition['type'] ?? 'string'; $format = $fieldDefinition['format'] ?? ''; - + // **FILE TYPE HANDLING**: File fields should use text_general for large content - if ($type === 'file' || $format === 'file' || $format === 'binary' - || in_array($format, ['data-url', 'base64', 'image', 'document']) - ) { - return 'text_general'; - // Large text type for file content + if ($type === 'file' || $format === 'file' || $format === 'binary' || + in_array($format, ['data-url', 'base64', 'image', 'document'])) { + return 'text_general'; // Large text type for file content } - + // Map OpenRegister types to SOLR types // Type determination should be based on data semantics, not facetability + // Note: 'file' type is handled above with early return return match ($type) { - 'string' => 'string', - // Exact values, IDs, codes, etc. - 'text' => 'text_general', - // Full-text searchable content + 'string' => 'string', // Exact values, IDs, codes, etc. + 'text' => 'text_general', // Full-text searchable content 'integer', 'int' => 'pint', - 'number', 'float', 'double' => 'pfloat', + 'number', 'float', 'double' => 'pfloat', 'boolean', 'bool' => 'boolean', 'date', 'datetime' => 'pdate', - 'array' => 'string', - // Multi-valued string (type=string, multiValued=true) - 'file' => 'text_general', - // File content (large text) + 'array' => 'string', // Multi-valued string (type=string, multiValued=true) default => 'string' }; - - }//end determineSolrFieldType() - + } /** * Check if field should be multi-valued based strictly on schema property type * * Only fields with type 'array' in the OpenRegister schema should be multi-valued in SOLR. - * This prevents issues where string fields incorrectly become multi-valued due to + * This prevents issues where string fields incorrectly become multi-valued due to * different SOLR configurations between environments. * - * @param array $fieldDefinition Field definition from OpenRegister schema + * @param array $fieldDefinition Field definition from OpenRegister schema * @return bool True if field should be multi-valued */ private function isMultiValued(array $fieldDefinition): bool { // STRICT: Only array type should be multi-valued return ($fieldDefinition['type'] ?? '') === 'array'; - - }//end isMultiValued() - + } /** * Determine if a core metadata field should be multi-valued @@ -991,28 +833,26 @@ private function isMultiValued(array $fieldDefinition): bool * Only specific core fields that legitimately store multiple values should be multi-valued. * This prevents accidental multi-value configuration for single-value fields. * - * @param string $fieldName Core field name - * @param string $fieldType SOLR field type + * @param string $fieldName Core field name + * @param string $fieldType SOLR field type (unused, kept for API compatibility) * @return bool True if field should be multi-valued + * + * @psalm-suppress UnusedParam */ private function isCoreFieldMultiValued(string $fieldName, string $fieldType): bool { // Only these core fields are legitimately multi-valued $multiValuedCoreFields = [ - 'self_relations', - // Array of UUID references - 'self_files', - // Array of file references - '_classification_', - // Auto-classification results (multi-valued strings) + 'self_relations', // Array of UUID references + 'self_files', // Array of file references + '_classification_' // Auto-classification results (multi-valued strings) ]; - + // NOTE: _embedding_ is NOT multi-valued! It's a single knn_vector (dense vector) // The vector itself contains an array of floats, but that's internal to the type + return in_array($fieldName, $multiValuedCoreFields); - - }//end isCoreFieldMultiValued() - + } /** * Determine if a core metadata field should be indexed @@ -1020,31 +860,23 @@ private function isCoreFieldMultiValued(string $fieldName, string $fieldType): b * Some core fields like JSON storage fields should be stored but not indexed * since they're only used for reconstruction, not searching. * - * @param string $fieldName Core field name + * @param string $fieldName Core field name * @return bool True if field should be indexed */ private function shouldCoreFieldBeIndexed(string $fieldName): bool { // Fields that should NOT be indexed (stored only for reconstruction) $nonIndexedFields = [ - 'self_object', - // JSON blob for object reconstruction - 'self_authorization', - // JSON blob for permissions - 'self_deleted', - // JSON blob for deletion metadata - 'self_validation', - // JSON blob for validation results - 'self_groups', - // JSON blob for group assignments - '_embedding_dim_', - // Dimension count - stored for validation, not searched + 'self_object', // JSON blob for object reconstruction + 'self_authorization', // JSON blob for permissions + 'self_deleted', // JSON blob for deletion metadata + 'self_validation', // JSON blob for validation results + 'self_groups', // JSON blob for group assignments + '_embedding_dim_' // Dimension count - stored for validation, not searched ]; - + return !in_array($fieldName, $nonIndexedFields); - - }//end shouldCoreFieldBeIndexed() - + } /** * Determine if a core metadata field should have docValues enabled @@ -1054,10 +886,10 @@ private function shouldCoreFieldBeIndexed(string $fieldName): bool * - Sorting (e.g., name, created, updated dates) * - Faceting (e.g., owner, organisation, schema, register) * - Grouping operations - * + * * JSON storage fields should have docValues=false to save storage space. * - * @param string $fieldName Core field name + * @param string $fieldName Core field name * @return bool True if field should have docValues enabled */ private function shouldCoreFieldHaveDocValues(string $fieldName): bool @@ -1065,135 +897,98 @@ private function shouldCoreFieldHaveDocValues(string $fieldName): bool // Fields that should have docValues enabled for sorting/faceting/grouping $docValuesFields = [ // Sortable fields - 'self_name', - // Sort by name - 'self_created', - // Sort by creation date - 'self_updated', - // Sort by update date - 'self_published', - // Sort by publication date + 'self_name', // Sort by name + 'self_created', // Sort by creation date + 'self_updated', // Sort by update date + 'self_published', // Sort by publication date + // Facetable fields - 'self_owner', - // Facet by owner - 'self_organisation', - // Facet by organisation - 'self_application', - // Facet by application - 'self_schema', - // Facet by schema ID - 'self_schema_id', - // Facet by schema ID - 'self_register', - // Facet by register ID - 'self_register_id', - // Facet by register ID + 'self_owner', // Facet by owner + 'self_organisation', // Facet by organisation + 'self_application', // Facet by application + 'self_schema', // Facet by schema ID + 'self_schema_id', // Facet by schema ID + 'self_register', // Facet by register ID + 'self_register_id', // Facet by register ID + // UUID fields for exact matching and grouping - 'self_uuid', - // Exact UUID matching - 'self_schema_uuid', - // Schema UUID matching - 'self_register_uuid', - // Register UUID matching + 'self_uuid', // Exact UUID matching + 'self_schema_uuid', // Schema UUID matching + 'self_register_uuid',// Register UUID matching + // Slug fields for URL-friendly lookups - 'self_slug', - // URL slug lookup - 'self_schema_slug', - // Schema slug lookup - 'self_register_slug', - // Register slug lookup + 'self_slug', // URL slug lookup + 'self_schema_slug', // Schema slug lookup + 'self_register_slug',// Register slug lookup + // Other metadata that might be used for filtering - 'self_object_id', - // Object ID filtering - 'self_tenant', - // Tenant filtering - 'self_version', - // Version filtering - 'self_size', - // Size-based sorting/filtering - 'self_locked', - // Locked status filtering + 'self_object_id', // Object ID filtering + 'self_tenant', // Tenant filtering + 'self_version', // Version filtering + 'self_size', // Size-based sorting/filtering + 'self_locked', // Locked status filtering ]; - + // Special handling for system fields if ($fieldName === '_text_') { - return false; - // Full-text search fields don't need docValues + return false; // Full-text search fields don't need docValues } - + // AI/ML fields configuration if (in_array($fieldName, ['_embedding_', '_confidence_', '_classification_'])) { - return false; - // Vector and classification fields don't need docValues for sorting + return false; // Vector and classification fields don't need docValues for sorting } - + if (in_array($fieldName, ['_embedding_model_', '_embedding_dim_', 'vector_indexed', 'vector_model', 'vector_dimensions'])) { - return true; - // Metadata fields that might be used for filtering/faceting + return true; // Metadata fields that might be used for filtering/faceting } - + return in_array($fieldName, $docValuesFields); - - }//end shouldCoreFieldHaveDocValues() - + } /** * Determine if a file metadata field should be multi-valued - * - * @param string $fieldName File field name - * @param string $fieldType Field type + * + * @param string $fieldName File field name * @return bool True if field should be multi-valued */ - private function isFileFieldMultiValued(string $fieldName, string $fieldType): bool + private function isFileFieldMultiValued(string $fieldName): bool { // Multi-valued file fields $multiValuedFileFields = [ - 'file_labels', - // User-defined labels - 'file_tags', - // Auto-generated tags - 'file_categories', - // Categories - 'shared_with', - // Users/groups with access - '_embedding_', - // Vector embeddings (multi-valued floats) - '_classification_', - // Auto-classification results + 'file_labels', // User-defined labels + 'file_tags', // Auto-generated tags + 'file_categories', // Categories + 'shared_with', // Users/groups with access + '_embedding_', // Vector embeddings (multi-valued floats) + '_classification_' // Auto-classification results ]; - + return in_array($fieldName, $multiValuedFileFields); - - }//end isFileFieldMultiValued() - + } /** * Determine if a file metadata field should be indexed - * - * @param string $fieldName File field name + * + * @param string $fieldName File field name * @return bool True if field should be indexed */ private function shouldFileFieldBeIndexed(string $fieldName): bool { // Fields that should NOT be indexed (stored only for metadata/reconstruction) $nonIndexedFields = [ - 'file_checksum', - // Only for deduplication, not searching - 'processing_error', - // Only for debugging, not searching - '_embedding_dim_', - // Dimension count - stored for validation, not searched + 'file_checksum', // Only for deduplication, not searching + 'processing_error', // Only for debugging, not searching + '_embedding_dim_' // Dimension count - stored for validation, not searched ]; - + return !in_array($fieldName, $nonIndexedFields); - - }//end shouldFileFieldBeIndexed() - + } /** * Determine if a file metadata field should have docValues enabled - * - * @param string $fieldName File field name + * + * @param string $fieldName File field name * @return bool True if field should have docValues enabled */ private function shouldFileFieldHaveDocValues(string $fieldName): bool @@ -1201,76 +996,52 @@ private function shouldFileFieldHaveDocValues(string $fieldName): bool // Fields that should have docValues enabled for sorting/faceting/grouping $docValuesFields = [ // Sortable fields - 'file_name', - // Sort by name - 'file_size', - // Sort by size - 'file_created', - // Sort by creation date - 'file_modified', - // Sort by modification date - 'chunk_index', - // Sort chunks by index - 'vector_updated', - // Sort by vector update date - 'processing_date', - // Sort by processing date + 'file_name', // Sort by name + 'file_size', // Sort by size + 'file_created', // Sort by creation date + 'file_modified', // Sort by modification date + 'chunk_index', // Sort chunks by index + 'vector_updated', // Sort by vector update date + 'processing_date', // Sort by processing date + // Facetable fields - 'file_extension', - // Facet by file type - 'file_mime_type', - // Facet by MIME type - 'file_owner', - // Facet by owner - 'file_labels', - // Facet by labels - 'file_tags', - // Facet by tags - 'file_categories', - // Facet by categories - 'file_language', - // Facet by language - 'processing_status', - // Facet by status - 'access_level', - // Facet by access level - 'extraction_method', - // Facet by extraction method + 'file_extension', // Facet by file type + 'file_mime_type', // Facet by MIME type + 'file_owner', // Facet by owner + 'file_labels', // Facet by labels + 'file_tags', // Facet by tags + 'file_categories', // Facet by categories + 'file_language', // Facet by language + 'processing_status', // Facet by status + 'access_level', // Facet by access level + 'extraction_method', // Facet by extraction method + // Vector metadata - 'vector_indexed', - // Filter by vectorization status - 'vector_model', - // Filter by model - 'vector_dimensions', - // Filter by dimensions + 'vector_indexed', // Filter by vectorization status + 'vector_model', // Filter by model + 'vector_dimensions', // Filter by dimensions + // Relationship fields - 'file_id', - // Nextcloud file ID lookup - 'related_object_id', - // Related object lookup - 'related_object_type', - // Related object type lookup + 'file_id', // Nextcloud file ID lookup + 'related_object_id', // Related object lookup + 'related_object_type',// Related object type lookup + // OCR fields - 'ocr_performed', - // Filter by OCR status + 'ocr_performed', // Filter by OCR status ]; - + // Special handling for system fields if (in_array($fieldName, ['_text_', 'chunk_text', 'text_content'])) { - return false; - // Full-text search fields don't need docValues + return false; // Full-text search fields don't need docValues } - + // AI/ML fields configuration if (in_array($fieldName, ['_embedding_', '_confidence_', '_classification_'])) { - return false; - // Vector and classification fields don't need docValues + return false; // Vector and classification fields don't need docValues } - + return in_array($fieldName, $docValuesFields); - - }//end shouldFileFieldHaveDocValues() - + } /** * Ensure core metadata fields exist in SOLR schema @@ -1278,39 +1049,32 @@ private function shouldFileFieldHaveDocValues(string $fieldName): bool * These are the essential fields needed for object indexing including * register and schema metadata (UUID, slug, etc.) * - * @param bool $force Force update existing fields + * @param bool $force Force update existing fields * @return bool Success status */ - private function ensureCoreMetadataFields(bool $force=false): bool + private function ensureCoreMetadataFields(bool $force = false): bool { - $this->logger->info( - '🔧 Ensuring core metadata fields in SOLR schema', - [ - 'field_count' => count(self::CORE_METADATA_FIELDS), - 'force' => $force, - ] - ); + $this->logger->info('🔧 Ensuring core metadata fields in SOLR schema', [ + 'field_count' => count(self::CORE_METADATA_FIELDS), + 'force' => $force + ]); // STEP 1: Ensure knn_vector field type exists (required for _embedding_ field) try { - $settings = $this->settingsService->getSettings(); + $settings = $this->settingsService->getSettings(); $objectCollection = $settings['solr']['objectCollection'] ?? $settings['solr']['collection'] ?? null; - $fileCollection = $settings['solr']['fileCollection'] ?? null; - + $fileCollection = $settings['solr']['fileCollection'] ?? null; + if ($objectCollection) { $this->ensureVectorFieldType($objectCollection, 4096, 'cosine'); } - if ($fileCollection) { $this->ensureVectorFieldType($fileCollection, 4096, 'cosine'); } } catch (\Exception $e) { - $this->logger->warning( - 'Failed to ensure knn_vector field type', - [ - 'error' => $e->getMessage(), - ] - ); + $this->logger->warning('Failed to ensure knn_vector field type', [ + 'error' => $e->getMessage() + ]); } // STEP 2: Ensure core metadata fields @@ -1318,107 +1082,82 @@ private function ensureCoreMetadataFields(bool $force=false): bool foreach (self::CORE_METADATA_FIELDS as $fieldName => $fieldType) { try { $fieldConfig = [ - 'type' => $fieldType, - 'stored' => true, - 'indexed' => $this->shouldCoreFieldBeIndexed($fieldName), - 'multiValued' => $this->isCoreFieldMultiValued($fieldName, $fieldType), - 'docValues' => $this->shouldCoreFieldHaveDocValues($fieldName), + 'type' => $fieldType, + 'stored' => true, + 'indexed' => $this->shouldCoreFieldBeIndexed($fieldName), + 'multiValued' => $this->isCoreFieldMultiValued($fieldName), + 'docValues' => $this->shouldCoreFieldHaveDocValues($fieldName) ]; if ($this->addOrUpdateSolrField($fieldName, $fieldConfig, $force)) { $successCount++; - $this->logger->debug( - '✅ Core metadata field ensured', - [ - 'field' => $fieldName, - 'type' => $fieldType, - ] - ); + $this->logger->debug('✅ Core metadata field ensured', [ + 'field' => $fieldName, + 'type' => $fieldType + ]); } } catch (\Exception $e) { - $this->logger->error( - '❌ Failed to ensure core metadata field', - [ - 'field' => $fieldName, - 'error' => $e->getMessage(), - ] - ); - }//end try - }//end foreach - - $this->logger->info( - 'Core metadata fields processing completed', - [ - 'successful' => $successCount, - 'total' => count(self::CORE_METADATA_FIELDS), - ] - ); - - return $successCount === count(self::CORE_METADATA_FIELDS); + $this->logger->error('❌ Failed to ensure core metadata field', [ + 'field' => $fieldName, + 'error' => $e->getMessage() + ]); + } + } - }//end ensureCoreMetadataFields() + $this->logger->info('Core metadata fields processing completed', [ + 'successful' => $successCount, + 'total' => count(self::CORE_METADATA_FIELDS) + ]); + return $successCount === count(self::CORE_METADATA_FIELDS); + } /** * Ensure file metadata fields exist in file collection - * - * @param bool $force Force update existing fields + * + * @param bool $force Force update existing fields * @return bool Success status */ - private function ensureFileMetadataFields(bool $force=false): bool + private function ensureFileMetadataFields(bool $force = false): bool { - $this->logger->info( - '🔧 Ensuring file metadata fields in SOLR schema', - [ - 'field_count' => count(self::FILE_METADATA_FIELDS), - 'force' => $force, - ] - ); + $this->logger->info('🔧 Ensuring file metadata fields in SOLR schema', [ + 'field_count' => count(self::FILE_METADATA_FIELDS), + 'force' => $force + ]); $successCount = 0; foreach (self::FILE_METADATA_FIELDS as $fieldName => $fieldType) { try { $fieldConfig = [ - 'type' => $fieldType, - 'stored' => true, - 'indexed' => $this->shouldFileFieldBeIndexed($fieldName), - 'multiValued' => $this->isFileFieldMultiValued($fieldName, $fieldType), - 'docValues' => $this->shouldFileFieldHaveDocValues($fieldName), + 'type' => $fieldType, + 'stored' => true, + 'indexed' => $this->shouldFileFieldBeIndexed($fieldName), + 'multiValued' => $this->isFileFieldMultiValued($fieldName), + 'docValues' => $this->shouldFileFieldHaveDocValues($fieldName) ]; if ($this->addOrUpdateSolrField($fieldName, $fieldConfig, $force)) { $successCount++; - $this->logger->debug( - '✅ File metadata field ensured', - [ - 'field' => $fieldName, - 'type' => $fieldType, - ] - ); + $this->logger->debug('✅ File metadata field ensured', [ + 'field' => $fieldName, + 'type' => $fieldType + ]); } } catch (\Exception $e) { - $this->logger->error( - '❌ Failed to ensure file metadata field', - [ - 'field' => $fieldName, - 'error' => $e->getMessage(), - ] - ); - }//end try - }//end foreach - - $this->logger->info( - 'File metadata fields processing completed', - [ - 'successful' => $successCount, - 'total' => count(self::FILE_METADATA_FIELDS), - ] - ); - - return $successCount === count(self::FILE_METADATA_FIELDS); + $this->logger->error('❌ Failed to ensure file metadata field', [ + 'field' => $fieldName, + 'error' => $e->getMessage() + ]); + } + } - }//end ensureFileMetadataFields() + $this->logger->info('File metadata fields processing completed', [ + 'successful' => $successCount, + 'total' => count(self::FILE_METADATA_FIELDS) + ]); + return $successCount === count(self::FILE_METADATA_FIELDS); + } /** * Get missing and extra fields in object collection @@ -1428,66 +1167,63 @@ private function ensureFileMetadataFields(bool $force=false): bool public function getObjectCollectionFieldStatus(): array { // Get object collection from settings - $settings = $this->settingsService->getSettings(); + $settings = $this->settingsService->getSettings(); $objectCollection = $settings['solr']['objectCollection'] ?? null; if (!$objectCollection) { // Fall back to default collection if object collection not configured $objectCollection = $settings['solr']['collection'] ?? 'openregister'; } - + // Get current fields from SOLR for object collection $current = $this->getCurrentCollectionFields($objectCollection); - + // Expected fields with their config - $expected = self::CORE_METADATA_FIELDS; + $expected = self::CORE_METADATA_FIELDS; $expectedNames = array_keys($expected); - + // Find missing fields (expected but not in SOLR) $missingNames = array_diff($expectedNames, $current); - $missing = []; + $missing = []; foreach ($missingNames as $fieldName) { $fieldType = $expected[$fieldName]; - + // Determine if field should be multi-valued (fields ending in _ss, _is, etc.) - $multiValued = str_ends_with($fieldName, '_ss') || - str_ends_with($fieldName, '_is') || + $multiValued = str_ends_with($fieldName, '_ss') || + str_ends_with($fieldName, '_is') || str_ends_with($fieldName, '_ls') || str_ends_with($fieldName, '_ts') || str_ends_with($fieldName, '_ds') || str_ends_with($fieldName, '_bs'); - + // Determine if field should have docValues (for sorting/faceting) // String fields and numeric fields typically need docValues for faceting $docValues = in_array($fieldType, ['string', 'pint', 'plong', 'pfloat', 'pdouble', 'pdate']) && - !str_starts_with($fieldName, 'self_object') && - // JSON storage fields don't need docValues + !str_starts_with($fieldName, 'self_object') && // JSON storage fields don't need docValues !str_starts_with($fieldName, 'self_schema') && !str_starts_with($fieldName, 'self_register') && !str_ends_with($fieldName, '_json'); - + $missing[$fieldName] = [ - 'type' => $fieldType, - 'stored' => true, - 'indexed' => true, + 'type' => $fieldType, + 'stored' => true, + 'indexed' => true, 'multiValued' => $multiValued, - 'docValues' => $docValues, + 'docValues' => $docValues ]; - }//end foreach - + } + // Find extra fields (in SOLR but not expected) $extra = array_diff($current, $expectedNames); - + return [ - 'missing' => $missing, - 'extra' => array_values($extra), - 'expected' => $expectedNames, - 'current' => $current, - 'status' => empty($missing) ? 'complete' : 'incomplete', - 'collection' => $objectCollection, + 'missing' => $missing, + 'extra' => array_values($extra), + 'expected' => $expectedNames, + 'current' => $current, + 'status' => empty($missing) ? 'complete' : 'incomplete', + 'collection' => $objectCollection ]; - - }//end getObjectCollectionFieldStatus() - + } /** * Get missing and extra fields in file collection @@ -1497,112 +1233,105 @@ public function getObjectCollectionFieldStatus(): array public function getFileCollectionFieldStatus(): array { // Get file collection from settings - $settings = $this->settingsService->getSettings(); + $settings = $this->settingsService->getSettings(); $fileCollection = $settings['solr']['fileCollection'] ?? null; if (!$fileCollection) { // File collection might not be configured yet $fileCollection = 'openregister_files'; } - + // Get current fields from SOLR for file collection $current = $this->getCurrentCollectionFields($fileCollection); - + // Expected fields with their config - $expected = self::FILE_METADATA_FIELDS; + $expected = self::FILE_METADATA_FIELDS; $expectedNames = array_keys($expected); - + // Find missing fields (expected but not in SOLR) $missingNames = array_diff($expectedNames, $current); - $missing = []; + $missing = []; foreach ($missingNames as $fieldName) { $fieldType = $expected[$fieldName]; - + // Determine if field should be multi-valued (fields ending in _ss, _is, etc.) - $multiValued = str_ends_with($fieldName, '_ss') || - str_ends_with($fieldName, '_is') || + $multiValued = str_ends_with($fieldName, '_ss') || + str_ends_with($fieldName, '_is') || str_ends_with($fieldName, '_ls') || str_ends_with($fieldName, '_ts') || str_ends_with($fieldName, '_ds') || str_ends_with($fieldName, '_bs'); - + // Determine if field should have docValues (for sorting/faceting) // String fields and numeric fields typically need docValues for faceting $docValues = in_array($fieldType, ['string', 'pint', 'plong', 'pfloat', 'pdouble', 'pdate']) && - !str_ends_with($fieldName, '_text') && - // Full-text fields don't need docValues + !str_ends_with($fieldName, '_text') && // Full-text fields don't need docValues !str_ends_with($fieldName, '_content') && !str_ends_with($fieldName, '_json'); - + $missing[$fieldName] = [ - 'type' => $fieldType, - 'stored' => true, - 'indexed' => true, + 'type' => $fieldType, + 'stored' => true, + 'indexed' => true, 'multiValued' => $multiValued, - 'docValues' => $docValues, + 'docValues' => $docValues ]; - }//end foreach - + } + // Find extra fields (in SOLR but not expected) $extra = array_diff($current, $expectedNames); - + return [ - 'missing' => $missing, - 'extra' => array_values($extra), - 'expected' => $expectedNames, - 'current' => $current, - 'status' => empty($missing) ? 'complete' : 'incomplete', - 'collection' => $fileCollection, + 'missing' => $missing, + 'extra' => array_values($extra), + 'expected' => $expectedNames, + 'current' => $current, + 'status' => empty($missing) ? 'complete' : 'incomplete', + 'collection' => $fileCollection ]; - - }//end getFileCollectionFieldStatus() - + } /** * Get current field names from a specific SOLR collection * * @param string $collectionName Collection to query - * + * * @return array Field names */ private function getCurrentCollectionFields(string $collectionName): array { try { // Build schema API URL for specific collection - $schemaUrl = $this->solrService->buildSolrBaseUrl()."/{$collectionName}/schema"; - + $schemaUrl = $this->solrService->buildSolrBaseUrl() . "/{$collectionName}/schema"; + // Prepare request options - $solrConfig = $this->settingsService->getSettings()['solr'] ?? []; + $solrConfig = $this->settingsService->getSettings()['solr'] ?? []; $requestOptions = [ 'timeout' => $solrConfig['timeout'] ?? 30, - 'headers' => ['Accept' => 'application/json'], + 'headers' => ['Accept' => 'application/json'] ]; // Add authentication if configured if (!empty($solrConfig['username']) && !empty($solrConfig['password'])) { $requestOptions['auth'] = [ $solrConfig['username'], - $solrConfig['password'], + $solrConfig['password'] ]; } // Make the schema request - $httpClient = \OC::$server->get(\OCP\Http\Client\IClientService::class)->newClient(); - $response = $httpClient->get($schemaUrl, $requestOptions); + $httpClient = \OC::$server->get(\OCP\Http\Client\IClientService::class)->newClient(); + $response = $httpClient->get($schemaUrl, $requestOptions); $responseBody = $response->getBody(); - $schemaData = json_decode($responseBody, true); + $schemaData = json_decode($responseBody, true); if (!$schemaData || !isset($schemaData['schema']['fields'])) { - $this->logger->warning( - 'No fields data returned from SOLR', - [ - 'collection' => $collectionName, - 'response' => substr($responseBody, 0, 500), - // Log first 500 chars for debugging - ] - ); + $this->logger->warning('No fields data returned from SOLR', [ + 'collection' => $collectionName, + 'response' => substr($responseBody, 0, 500) // Log first 500 chars for debugging + ]); return []; } - + // Extract field names $fieldNames = []; foreach ($schemaData['schema']['fields'] as $field) { @@ -1610,119 +1339,100 @@ private function getCurrentCollectionFields(string $collectionName): array $fieldNames[] = $field['name']; } } - + return $fieldNames; + } catch (\Exception $e) { - $this->logger->error( - 'Failed to get current collection fields', - [ - 'collection' => $collectionName, - 'error' => $e->getMessage(), - ] - ); + $this->logger->error('Failed to get current collection fields', [ + 'collection' => $collectionName, + 'error' => $e->getMessage() + ]); return []; - }//end try - - }//end getCurrentCollectionFields() - + } + } /** * Apply SOLR fields to the tenant collection using Schema API * - * @param array $solrFields SOLR field definitions - * @param bool $force Force update existing fields + * @param array $solrFields SOLR field definitions + * @param bool $force Force update existing fields * @return bool Success status */ - private function applySolrFields(array $solrFields, bool $force=false): bool + private function applySolrFields(array $solrFields, bool $force = false): bool { - $this->logger->info( - '🔧 Applying SOLR fields via Schema API', - [ - 'app' => 'openregister', - 'field_count' => count($solrFields), - 'fields' => array_keys($solrFields), - 'force' => $force, - ] - ); + $this->logger->info('🔧 Applying SOLR fields via Schema API', [ + 'app' => 'openregister', + 'field_count' => count($solrFields), + 'fields' => array_keys($solrFields), + 'force' => $force + ]); $successCount = 0; foreach ($solrFields as $fieldName => $fieldConfig) { try { if ($this->addOrUpdateSolrField($fieldName, $fieldConfig, $force)) { $successCount++; - $this->logger->info( - '✅ Applied SOLR field', - [ - 'field' => $fieldName, - 'type' => $fieldConfig['type'], - ] - ); + $this->logger->info('✅ Applied SOLR field', [ + 'field' => $fieldName, + 'type' => $fieldConfig['type'] + ]); // DEBUG: Special logging for versie field if ($fieldName === 'versie') { $this->logger->debug('=== VERSIE FIELD CREATED ==='); - $this->logger->debug('Field: '.$fieldName); - $this->logger->debug('Type: '.$fieldConfig['type']); - $this->logger->debug('Config: '.json_encode($fieldConfig)); + $this->logger->debug('Field: ' . $fieldName); + $this->logger->debug('Type: ' . $fieldConfig['type']); + $this->logger->debug('Config: ' . json_encode($fieldConfig)); $this->logger->debug('=== END VERSIE DEBUG ==='); } } } catch (\Exception $e) { - $this->logger->error( - '❌ Failed to apply SOLR field', - [ - 'field' => $fieldName, - 'error' => $e->getMessage(), - ] - ); - }//end try - }//end foreach - - $this->logger->info( - 'Schema field application completed', - [ - 'successful' => $successCount, - 'total' => count($solrFields), - ] - ); - - return $successCount === count($solrFields); + $this->logger->error('❌ Failed to apply SOLR field', [ + 'field' => $fieldName, + 'error' => $e->getMessage() + ]); + } + } - }//end applySolrFields() + $this->logger->info('Schema field application completed', [ + 'successful' => $successCount, + 'total' => count($solrFields) + ]); + return $successCount === count($solrFields); + } /** * Add or update a single SOLR field using Schema API * - * @param string $fieldName Field name - * @param array $fieldConfig Field configuration - * @param bool $force Force update existing fields + * @param string $fieldName Field name + * @param array $fieldConfig Field configuration + * @param bool $force Force update existing fields * @return bool Success status */ - private function addOrUpdateSolrField(string $fieldName, array $fieldConfig, bool $force=false): bool + private function addOrUpdateSolrField(string $fieldName, array $fieldConfig, bool $force = false): bool { // Get SOLR settings - $solrConfig = $this->settingsService->getSolrSettings(); + $solrConfig = $this->settingsService->getSolrSettings(); $baseCollectionName = $solrConfig['core'] ?? 'openregister'; - + // Build SOLR URL - handle Kubernetes service names properly $host = $solrConfig['host'] ?? 'localhost'; $port = $solrConfig['port'] ?? null; - + // Normalize port - convert string '0' to null, handle empty strings if ($port === '0' || $port === '' || $port === null) { $port = null; } else { - $port = (int) $port; + $port = (int)$port; if ($port === 0) { $port = null; } } - + // Check if it's a Kubernetes service name (contains .svc.cluster.local) if (strpos($host, '.svc.cluster.local') !== false) { // Kubernetes service - don't append port, it's handled by the service - $url = sprintf( - '%s://%s%s/%s/schema', + $url = sprintf('%s://%s%s/%s/schema', $solrConfig['scheme'] ?? 'http', $host, $solrConfig['path'] ?? '/solr', @@ -1731,8 +1441,7 @@ private function addOrUpdateSolrField(string $fieldName, array $fieldConfig, boo } else { // Regular hostname - only append port if explicitly provided and not 0/null if ($port !== null && $port > 0) { - $url = sprintf( - '%s://%s:%d%s/%s/schema', + $url = sprintf('%s://%s:%d%s/%s/schema', $solrConfig['scheme'] ?? 'http', $host, $port, @@ -1741,19 +1450,18 @@ private function addOrUpdateSolrField(string $fieldName, array $fieldConfig, boo ); } else { // No port provided - let the service handle it - $url = sprintf( - '%s://%s%s/%s/schema', + $url = sprintf('%s://%s%s/%s/schema', $solrConfig['scheme'] ?? 'http', $host, $solrConfig['path'] ?? '/solr', $baseCollectionName ); } - }//end if + } // Try to add field first $payload = [ - 'add-field' => array_merge(['name' => $fieldName], $fieldConfig), + 'add-field' => array_merge(['name' => $fieldName], $fieldConfig) ]; if ($this->makeSolrSchemaRequest($url, $payload)) { @@ -1763,35 +1471,31 @@ private function addOrUpdateSolrField(string $fieldName, array $fieldConfig, boo // If add failed and force is enabled, try to replace if ($force) { $payload = [ - 'replace-field' => array_merge(['name' => $fieldName], $fieldConfig), + 'replace-field' => array_merge(['name' => $fieldName], $fieldConfig) ]; return $this->makeSolrSchemaRequest($url, $payload); } return false; - - }//end addOrUpdateSolrField() - + } /** * Make HTTP request to SOLR Schema API * - * @param string $url SOLR schema endpoint URL - * @param array $payload Request payload + * @param string $url SOLR schema endpoint URL + * @param array $payload Request payload * @return bool Success status */ private function makeSolrSchemaRequest(string $url, array $payload): bool { - $context = stream_context_create( - [ - 'http' => [ - 'method' => 'POST', - 'header' => 'Content-Type: application/json', - 'content' => json_encode($payload), - 'timeout' => 30, - ], - ] - ); + $context = stream_context_create([ + 'http' => [ + 'method' => 'POST', + 'header' => 'Content-Type: application/json', + 'content' => json_encode($payload), + 'timeout' => 30 + ] + ]); $response = @file_get_contents($url, false, $context); if ($response === false) { @@ -1800,9 +1504,7 @@ private function makeSolrSchemaRequest(string $url, array $payload): bool $data = json_decode($response, true); return ($data['responseHeader']['status'] ?? -1) === 0; - - }//end makeSolrSchemaRequest() - + } /** * Get schema mirroring statistics @@ -1812,30 +1514,28 @@ private function makeSolrSchemaRequest(string $url, array $payload): bool public function getSchemaStats(): array { try { - $tenantId = $this->settingsService->getTenantId(); + $tenantId = $this->settingsService->getTenantId(); $organisationId = $this->settingsService->getOrganisationId(); - + // Get schema counts $schemaCount = $this->schemaMapper->findAll(null, null, [$organisationId]); - + return [ - 'success' => true, - 'tenant_id' => $tenantId, - 'organisation_id' => $organisationId, + 'success' => true, + 'tenant_id' => $tenantId, + 'organisation_id' => $organisationId, 'openregister_schemas' => count($schemaCount), - 'solr_collection' => $this->solrService->getTenantCollectionName(), - 'last_sync' => null, - // TODO: Track last sync time + 'solr_collection' => $this->solrService->getTenantCollectionName(), + 'last_sync' => null // TODO: Track last sync time ]; + } catch (\Exception $e) { return [ 'success' => false, - 'error' => $e->getMessage(), + 'error' => $e->getMessage() ]; - }//end try - - }//end getSchemaStats() - + } + } /** * Create missing fields in a specific collection @@ -1846,31 +1546,30 @@ public function getSchemaStats(): array * * @return array Creation result with statistics */ - public function createMissingFields(string $collectionType, array $missingFields, bool $dryRun=false): array + public function createMissingFields(string $collectionType, array $missingFields, bool $dryRun = false): array { - $this->logger->info( - 'Creating missing fields for collection', - [ - 'collection_type' => $collectionType, - 'field_count' => count($missingFields), - 'dry_run' => $dryRun, - ] - ); + $this->logger->info('Creating missing fields for collection', [ + 'collection_type' => $collectionType, + 'field_count' => count($missingFields), + 'dry_run' => $dryRun + ]); $startTime = microtime(true); - $created = []; - $errors = []; + $created = []; + $errors = []; // Get the appropriate collection name - $settings = $this->settingsService->getSettings(); - $collection = $collectionType === 'files' ? ($settings['solr']['fileCollection'] ?? null) : ($settings['solr']['objectCollection'] ?? $settings['solr']['collection'] ?? 'openregister'); + $settings = $this->settingsService->getSettings(); + $collection = $collectionType === 'files' + ? ($settings['solr']['fileCollection'] ?? null) + : ($settings['solr']['objectCollection'] ?? $settings['solr']['collection'] ?? 'openregister'); if (!$collection) { return [ - 'success' => false, - 'message' => "No collection configured for type: {$collectionType}", + 'success' => false, + 'message' => "No collection configured for type: {$collectionType}", 'created_count' => 0, - 'error_count' => 1, + 'error_count' => 1 ]; } @@ -1890,73 +1589,65 @@ public function createMissingFields(string $collectionType, array $missingFields if ($result) { $created[] = $fieldName; - $this->logger->debug( - 'Created field in SOLR', - [ - 'field' => $fieldName, - 'collection' => $collection, - ] - ); + $this->logger->debug('Created field in SOLR', [ + 'field' => $fieldName, + 'collection' => $collection + ]); } else { $errors[$fieldName] = 'Failed to create field'; } } catch (\Exception $e) { $errors[$fieldName] = $e->getMessage(); - $this->logger->error( - 'Failed to create field', - [ - 'field' => $fieldName, - 'collection' => $collection, - 'error' => $e->getMessage(), - ] - ); - }//end try - }//end foreach + $this->logger->error('Failed to create field', [ + 'field' => $fieldName, + 'collection' => $collection, + 'error' => $e->getMessage() + ]); + } + } $executionTime = round((microtime(true) - $startTime) * 1000, 2); return [ - 'success' => empty($errors), - 'message' => sprintf( + 'success' => empty($errors), + 'message' => sprintf( '%s: %d created, %d errors', $dryRun ? 'Dry run' : 'Created fields', count($created), count($errors) ), - 'collection' => $collection, - 'collection_type' => $collectionType, - 'created' => $created, - 'created_count' => count($created), - 'errors' => $errors, - 'error_count' => count($errors), + 'collection' => $collection, + 'collection_type' => $collectionType, + 'created' => $created, + 'created_count' => count($created), + 'errors' => $errors, + 'error_count' => count($errors), 'execution_time_ms' => $executionTime, - 'dry_run' => $dryRun, + 'dry_run' => $dryRun ]; - - }//end createMissingFields() - + } /** * Add a field to a SOLR collection using the Schema API * - * @param string $collection Collection name - * @param string $fieldName Field name + * @param string $collection Collection name + * @param string $fieldName Field name * @param array $fieldConfig Field configuration * * @return bool True if successful */ private function addFieldToCollection(string $collection, string $fieldName, array $fieldConfig): bool { - $settings = $this->settingsService->getSettings(); - $solrUrl = $this->solrService->buildSolrBaseUrl(); + $settings = $this->settingsService->getSettings(); + $solrUrl = $this->solrService->buildSolrBaseUrl(); $schemaUrl = "{$solrUrl}/{$collection}/schema"; // Prepare field definition $fieldDef = [ - 'name' => $fieldName, - 'type' => $fieldConfig['type'], - 'stored' => $fieldConfig['stored'] ?? true, - 'indexed' => $fieldConfig['indexed'] ?? true, + 'name' => $fieldName, + 'type' => $fieldConfig['type'], + 'stored' => $fieldConfig['stored'] ?? true, + 'indexed' => $fieldConfig['indexed'] ?? true ]; // Add multiValued if specified @@ -1973,12 +1664,12 @@ private function addFieldToCollection(string $collection, string $fieldName, arr // Prepare request options $requestOptions = [ - 'body' => json_encode($payload), + 'body' => json_encode($payload), 'timeout' => 30, 'headers' => [ 'Content-Type' => 'application/json', - 'Accept' => 'application/json', - ], + 'Accept' => 'application/json' + ] ]; // Add authentication if configured @@ -1990,25 +1681,19 @@ private function addFieldToCollection(string $collection, string $fieldName, arr try { // Get HTTP client from server - $httpClient = \OC::$server->get(\OCP\Http\Client\IClientService::class)->newClient(); - $response = $httpClient->post($schemaUrl, $requestOptions); + $httpClient = \OC::$server->get(\OCP\Http\Client\IClientService::class)->newClient(); + $response = $httpClient->post($schemaUrl, $requestOptions); $responseBody = $response->getBody(); - $data = json_decode($responseBody, true); - + $data = json_decode($responseBody, true); + return ($data['responseHeader']['status'] ?? -1) === 0; } catch (\Exception $e) { - $this->logger->error( - 'Failed to add field to collection', - [ - 'collection' => $collection, - 'field' => $fieldName, - 'error' => $e->getMessage(), - ] - ); + $this->logger->error('Failed to add field to collection', [ + 'collection' => $collection, + 'field' => $fieldName, + 'error' => $e->getMessage() + ]); return false; } - - }//end addFieldToCollection() - - -}//end class + } +} diff --git a/lib/Service/WebhookService.php b/lib/Service/WebhookService.php index 571b99bde..515c9ee9b 100644 --- a/lib/Service/WebhookService.php +++ b/lib/Service/WebhookService.php @@ -207,29 +207,65 @@ public function deliverWebhook(Webhook $webhook, string $eventName, array $paylo return true; } catch (RequestException $e) { - // Log failure. - $webhookLog->setSuccess(false); - $webhookLog->setErrorMessage($e->getMessage()); + // Build detailed error message from Guzzle exception. + $errorMessage = $e->getMessage(); + $errorDetails = []; // Get status code from exception if available. if ($e->hasResponse() === true) { - $webhookLog->setStatusCode($e->getResponse()->getStatusCode()); + $response = $e->getResponse(); + $statusCode = $response->getStatusCode(); + $webhookLog->setStatusCode($statusCode); + $errorDetails['status_code'] = $statusCode; + try { - $webhookLog->setResponseBody((string) $e->getResponse()->getBody()); + $responseBody = (string) $response->getBody(); + $webhookLog->setResponseBody($responseBody); + $errorDetails['response_body'] = $responseBody; + + // Try to parse JSON response for better error message. + $jsonResponse = json_decode($responseBody, true); + if ($jsonResponse !== null && isset($jsonResponse['message'])) { + $errorMessage .= ': '.$jsonResponse['message']; + } elseif ($jsonResponse !== null && isset($jsonResponse['error'])) { + $errorMessage .= ': '.$jsonResponse['error']; + } } catch (\Exception $bodyException) { // Ignore body reading errors. } + } else { + // Connection error or timeout. + $errorDetails['connection_error'] = true; + if ($e->getCode() !== 0) { + $errorDetails['error_code'] = $e->getCode(); + } } + // Add request details to error message. + $errorDetails['request_url'] = $webhook->getUrl(); + $errorDetails['request_method'] = $webhook->getMethod(); + $errorDetails['timeout'] = $webhook->getTimeout(); + + // Store request body as JSON for retry purposes (only on failure). + $webhookLog->setRequestBody(json_encode($webhookPayload)); + + // Log failure with detailed context. + $webhookLog->setSuccess(false); + $webhookLog->setErrorMessage($errorMessage); + $this->logger->error( message: 'Webhook delivery failed', context: [ 'webhook_id' => $webhook->getId(), 'webhook_name' => $webhook->getName(), 'event' => $eventName, - 'error' => $e->getMessage(), + 'error' => $errorMessage, + 'error_details' => $errorDetails, 'attempt' => $attempt, 'max_retries' => $webhook->getMaxRetries(), + 'exception_class' => get_class($e), + 'exception_code' => $e->getCode(), + 'trace' => $e->getTraceAsString(), ] ); @@ -380,10 +416,17 @@ private function sendRequest(Webhook $webhook, array $payload): array $options = [ 'headers' => $headers, - 'json' => $payload, 'timeout' => $webhook->getTimeout(), ]; + // For GET requests, use query parameters instead of JSON body. + if (strtoupper($webhook->getMethod()) === 'GET') { + $options['query'] = $payload; + } else { + // For POST, PUT, PATCH, DELETE, send JSON body. + $options['json'] = $payload; + } + $response = $this->client->request( method: $webhook->getMethod(), uri: $webhook->getUrl(), diff --git a/lib/Setup/apply_solr_schema.php b/lib/Setup/apply_solr_schema.php index 279db7920..59cef31d4 100644 --- a/lib/Setup/apply_solr_schema.php +++ b/lib/Setup/apply_solr_schema.php @@ -30,6 +30,10 @@ echo "Exiting without making changes...\n"; exit(1); +/* + * DEPRECATED CODE BELOW - This code is unreachable due to exit(1) above. + * Kept for reference only. Do not use this script. + * // Get collection name from command line or use default. $collectionName = $argv[1] ?? 'openregister_nc_f0e53393'; $solrBaseUrl = 'http://nextcloud-dev-solr:8983/solr'; @@ -391,3 +395,4 @@ function makeHttpRequest($url, $payload): array echo "⚠️ Some fields failed to configure. Check SOLR logs for details.\n"; exit(1); } +*/ diff --git a/src/modals/Modals.vue b/src/modals/Modals.vue index c07ce02b5..dc789c295 100644 --- a/src/modals/Modals.vue +++ b/src/modals/Modals.vue @@ -53,6 +53,7 @@ import { navigationStore } from '../store/store.js' + @@ -104,6 +105,7 @@ import EditApplication from './application/EditApplication.vue' import EditAgent from './agent/EditAgent.vue' import DeleteAgent from './agent/DeleteAgent.vue' import EditWebhook from './webhook/EditWebhook.vue' +import ViewWebhookLog from './webhook/ViewWebhookLog.vue' export default { name: 'Modals', components: { @@ -154,6 +156,7 @@ export default { EditAgent, DeleteAgent, EditWebhook, + ViewWebhookLog, }, } diff --git a/src/modals/register/ImportRegister.vue b/src/modals/register/ImportRegister.vue index 308e4357c..058de3a5d 100644 --- a/src/modals/register/ImportRegister.vue +++ b/src/modals/register/ImportRegister.vue @@ -37,8 +37,8 @@ import { registerStore, schemaStore, navigationStore, objectStore, dashboardStor - @@ -222,6 +231,7 @@ import PauseCircleOutline from 'vue-material-design-icons/PauseCircleOutline.vue import Pencil from 'vue-material-design-icons/Pencil.vue' import DeleteOutline from 'vue-material-design-icons/DeleteOutline.vue' import Plus from 'vue-material-design-icons/Plus.vue' +import FileDocumentOutline from 'vue-material-design-icons/FileDocumentOutline.vue' /** * Main view for managing webhooks @@ -243,6 +253,7 @@ export default { Pencil, DeleteOutline, Plus, + FileDocumentOutline, WebhooksSidebar, }, data() { @@ -279,7 +290,7 @@ export default { /** * Get properties for selected events * - * @return {array} Array of property options + * @return {Array} Array of property options */ selectedEventProperties() { if (!this.newWebhook.events || this.newWebhook.events.length === 0) { @@ -424,16 +435,38 @@ export default { */ async testWebhook(webhookId) { try { - await axios.post( + const response = await axios.post( generateUrl(`/apps/openregister/api/webhooks/${webhookId}/test`), ) - showSuccess(t('openregister', 'Test webhook sent successfully')) + // Check if the test was successful based on response data. + if (response.data && response.data.success === true) { + showSuccess(t('openregister', 'Test webhook sent successfully')) + } else { + const message = response.data?.message || response.data?.error || t('openregister', 'Test webhook delivery failed') + showError(message) + } + // Always refresh webhook list to show updated statistics (last triggered, success rate). + this.loadWebhooks() } catch (error) { console.error('Failed to test webhook:', error) - showError(t('openregister', 'Failed to test webhook')) + const errorMessage = error.response?.data?.error || error.response?.data?.message || t('openregister', 'Failed to test webhook') + showError(errorMessage) + // Refresh even on error to show any partial updates. + this.loadWebhooks() } }, + /** + * View logs for a webhook + * + * @param {number} webhookId - Webhook ID + * @return {void} + */ + viewLogs(webhookId) { + navigationStore.setTransferData({ webhookId }) + this.$router.push('/webhooks/logs') + }, + /** * Toggle webhook enabled status * @@ -473,7 +506,6 @@ export default { } }, - /** * Open create webhook dialog * @@ -594,6 +626,22 @@ export default { border-radius: var(--border-radius-large); overflow-x: auto; overflow-y: visible; + min-height: 200px; +} + +.tableContainer.is-loading { + overflow: hidden; + display: flex; + align-items: center; + justify-content: center; +} + +.loadingWrapper { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + padding: 40px; } .webhooksTable {