Pagable and efficient search

app/Entities/Models/Entity.php

@@ -471,4 +471,17 @@ protected function getContentsAttributes(): array
 
         return $contentFields;
     }
+
+    /**
+     * Create a new instance for the given entity type.
+     */
+    public static function instanceFromType(string $type): self
+    {
+        return match ($type) {
+            'page' => new Page(),
+            'chapter' => new Chapter(),
+            'book' => new Book(),
+            'bookshelf' => new Bookshelf(),
+        };
+    }
 }

app/Entities/Models/EntityTable.php

@@ -0,0 +1,69 @@
+<?php
+
+namespace BookStack\Entities\Models;
+
+use BookStack\Activity\Models\Tag;
+use BookStack\Activity\Models\View;
+use BookStack\App\Model;
+use BookStack\Permissions\Models\EntityPermission;
+use BookStack\Permissions\Models\JointPermission;
+use BookStack\Permissions\PermissionApplicator;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Relations\HasMany;
+use Illuminate\Database\Eloquent\Relations\MorphMany;
+use Illuminate\Database\Eloquent\SoftDeletes;
+
+/**
+ * This is a simplistic model interpretation of a generic Entity used to query and represent
+ * that database abstractly. Generally, this should rarely be used outside queries.
+ */
+class EntityTable extends Model
+{
+    use SoftDeletes;
+
+    protected $table = 'entities';
+
+    /**
+     * Get the entities that are visible to the current user.
+     */
+    public function scopeVisible(Builder $query): Builder
+    {
+        return app()->make(PermissionApplicator::class)->restrictEntityQuery($query);
+    }
+
+    /**
+     * Get the entity jointPermissions this is connected to.
+     */
+    public function jointPermissions(): HasMany
+    {
+        return $this->hasMany(JointPermission::class, 'entity_id')
+            ->whereColumn('entity_type', '=', 'entities.type');
+    }
+
+    /**
+     * Get the Tags that have been assigned to entities.
+     */
+    public function tags(): HasMany
+    {
+        return $this->hasMany(Tag::class, 'entity_id')
+            ->whereColumn('entity_type', '=', 'entities.type');
+    }
+
+    /**
+     * Get the assigned permissions.
+     */
+    public function permissions(): HasMany
+    {
+        return $this->hasMany(EntityPermission::class, 'entity_id')
+            ->whereColumn('entity_type', '=', 'entities.type');
+    }
+
+    /**
+     * Get View objects for this entity.
+     */
+    public function views(): HasMany
+    {
+        return $this->hasMany(View::class, 'viewable_id')
+            ->whereColumn('viewable_type', '=', 'entities.type');
+    }
+}

app/Entities/Queries/EntityQueries.php

@@ -3,7 +3,11 @@
 namespace BookStack\Entities\Queries;
 
 use BookStack\Entities\Models\Entity;
+use BookStack\Entities\Models\EntityTable;
 use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Query\Builder as QueryBuilder;
+use Illuminate\Database\Query\JoinClause;
+use Illuminate\Support\Facades\DB;
 use InvalidArgumentException;
 
 class EntityQueries
@@ -32,12 +36,37 @@ public function findVisibleByStringIdentifier(string $identifier): ?Entity
         return $queries->findVisibleById($entityId);
     }
 
+    /**
+     * Start a query across all entity types.
+     * Combines the description/text fields into a single 'description' field.
+     * @return Builder<EntityTable>
+     */
+    public function visibleForList(): Builder
+    {
+        $rawDescriptionField = DB::raw('COALESCE(description, text) as description');
+        $bookSlugSelect = function (QueryBuilder $query) {
+            return $query->select('slug')->from('entities as books')
+                ->whereColumn('books.id', '=', 'entities.book_id')
+                ->where('type', '=', 'book');
+        };
+
+        return EntityTable::query()->scopes('visible')
+            ->select(['id', 'type', 'name', 'slug', 'book_id', 'chapter_id', 'created_at', 'updated_at', 'draft', 'book_slug' => $bookSlugSelect, $rawDescriptionField])
+            ->leftJoin('entity_container_data', function (JoinClause $join) {
+                $join->on('entity_container_data.entity_id', '=', 'entities.id')
+                    ->on('entity_container_data.entity_type', '=', 'entities.type');
+            })->leftJoin('entity_page_data', function (JoinClause $join) {
+                $join->on('entity_page_data.page_id', '=', 'entities.id')
+                    ->where('entities.type', '=', 'page');
+            });
+    }
+
     /**
      * Start a query of visible entities of the given type,
      * suitable for listing display.
      * @return Builder<Entity>
      */
-    public function visibleForList(string $entityType): Builder
+    public function visibleForListForType(string $entityType): Builder
     {
         $queries = $this->getQueriesForType($entityType);
         return $queries->visibleForList();
@@ -48,7 +77,7 @@ public function visibleForList(string $entityType): Builder
      * suitable for using the contents of the items.
      * @return Builder<Entity>
      */
-    public function visibleForContent(string $entityType): Builder
+    public function visibleForContentForType(string $entityType): Builder
     {
         $queries = $this->getQueriesForType($entityType);
         return $queries->visibleForContent();

app/Entities/Tools/EntityHydrator.php

@@ -0,0 +1,140 @@
+<?php
+
+namespace BookStack\Entities\Tools;
+
+use BookStack\Activity\Models\Tag;
+use BookStack\Entities\Models\Chapter;
+use BookStack\Entities\Models\Entity;
+use BookStack\Entities\Models\EntityTable;
+use BookStack\Entities\Models\Page;
+use BookStack\Entities\Queries\EntityQueries;
+use Illuminate\Database\Eloquent\Collection;
+
+class EntityHydrator
+{
+    public function __construct(
+        protected EntityQueries $entityQueries,
+    ) {
+    }
+
+    /**
+     * Hydrate the entities of this hydrator to return a list of entities represented
+     * in their original intended models.
+     * @param EntityTable[] $entities
+     * @return Entity[]
+     */
+    public function hydrate(array $entities, bool $loadTags = false, bool $loadParents = false): array
+    {
+        $hydrated = [];
+
+        foreach ($entities as $entity) {
+            $data = $entity->getRawOriginal();
+            $instance = Entity::instanceFromType($entity->type);
+
+            if ($instance instanceof Page) {
+                $data['text'] = $data['description'];
+                unset($data['description']);
+            }
+
+            $instance = $instance->setRawAttributes($data, true);
+            $hydrated[] = $instance;
+        }
+
+        if ($loadTags) {
+            $this->loadTagsIntoModels($hydrated);
+        }
+
+        if ($loadParents) {
+            $this->loadParentsIntoModels($hydrated);
+        }
+
+        return $hydrated;
+    }
+
+    /**
+     * @param Entity[] $entities
+     */
+    protected function loadTagsIntoModels(array $entities): void
+    {
+        $idsByType = [];
+        $entityMap = [];
+        foreach ($entities as $entity) {
+            if (!isset($idsByType[$entity->type])) {
+                $idsByType[$entity->type] = [];
+            }
+            $idsByType[$entity->type][] = $entity->id;
+            $entityMap[$entity->type . ':' . $entity->id] = $entity;
+        }
+
+        $query = Tag::query();
+        foreach ($idsByType as $type => $ids) {
+            $query->orWhere(function ($query) use ($type, $ids) {
+                $query->where('entity_type', '=', $type)
+                    ->whereIn('entity_id', $ids);
+            });
+        }
+
+        $tags = empty($idsByType) ? [] : $query->get()->all();
+        $tagMap = [];
+        foreach ($tags as $tag) {
+            $key = $tag->entity_type . ':' . $tag->entity_id;
+            if (!isset($tagMap[$key])) {
+                $tagMap[$key] = [];
+            }
+            $tagMap[$key][] = $tag;
+        }
+
+        foreach ($entityMap as $key => $entity) {
+            $entityTags = new Collection($tagMap[$key] ?? []);
+            $entity->setRelation('tags', $entityTags);
+        }
+    }
+
+    /**
+     * @param Entity[] $entities
+     */
+    protected function loadParentsIntoModels(array $entities): void
+    {
+        $parentsByType = ['book' => [], 'chapter' => []];
+
+        foreach ($entities as $entity) {
+            if ($entity->getAttribute('book_id') !== null) {
+                $parentsByType['book'][] = $entity->getAttribute('book_id');
+            }
+            if ($entity->getAttribute('chapter_id') !== null) {
+                $parentsByType['chapter'][] = $entity->getAttribute('chapter_id');
+            }
+        }
+
+        $parentQuery = $this->entityQueries->visibleForList();
+        $filtered = count($parentsByType['book']) > 0 || count($parentsByType['chapter']) > 0;
+        $parentQuery = $parentQuery->where(function ($query) use ($parentsByType) {
+            foreach ($parentsByType as $type => $ids) {
+                if (count($ids) > 0) {
+                    $query = $query->orWhere(function ($query) use ($type, $ids) {
+                        $query->where('type', '=', $type)
+                            ->whereIn('id', $ids);
+                    });
+                }
+            }
+        });
+
+        $parentModels = $filtered ? $parentQuery->get()->all() : [];
+        $parents = $this->hydrate($parentModels);
+        $parentMap = [];
+        foreach ($parents as $parent) {
+            $parentMap[$parent->type . ':' . $parent->id] = $parent;
+        }
+
+        foreach ($entities as $entity) {
+            if ($entity instanceof Page || $entity instanceof Chapter) {
+                $key = 'book:' . $entity->getRawAttribute('book_id');
+                $entity->setRelation('book', $parentMap[$key] ?? null);
+            }
+            if ($entity instanceof Page) {
+                $key = 'chapter:' . $entity->getRawAttribute('chapter_id');
+                $entity->setRelation('chapter', $parentMap[$key] ?? null);
+            }
+        }
+    }
+}

app/Entities/Tools/MixedEntityListLoader.php

@@ -54,7 +54,7 @@ protected function idsByTypeToModelMap(array $idsByType, bool $eagerLoadParents,
         $modelMap = [];
 
         foreach ($idsByType as $type => $ids) {
-            $base = $withContents ? $this->queries->visibleForContent($type) : $this->queries->visibleForList($type);
+            $base = $withContents ? $this->queries->visibleForContentForType($type) : $this->queries->visibleForListForType($type);
             $models = $base->whereIn('id', $ids)
                 ->with($eagerLoadParents ? $this->getRelationsToEagerLoad($type) : [])
                 ->get();

app/Search/SearchApiController.php

@@ -1,10 +1,13 @@
 <?php
 
+declare(strict_types=1);
+
 namespace BookStack\Search;
 
 use BookStack\Api\ApiEntityListFormatter;
 use BookStack\Entities\Models\Entity;
 use BookStack\Http\ApiController;
+use Illuminate\Http\JsonResponse;
 use Illuminate\Http\Request;
 
 class SearchApiController extends ApiController
@@ -31,11 +34,9 @@ public function __construct(
      * between: bookshelf, book, chapter & page.
      *
      * The paging parameters and response format emulates a standard listing endpoint
-     * but standard sorting and filtering cannot be done on this endpoint. If a count value
-     * is provided this will only be taken as a suggestion. The results in the response
-     * may currently be up to 4x this value.
+     * but standard sorting and filtering cannot be done on this endpoint.
      */
-    public function all(Request $request)
+    public function all(Request $request): JsonResponse
     {
         $this->validate($request, $this->rules['all']);
 

app/Search/SearchController.php

@@ -7,6 +7,7 @@
 use BookStack\Entities\Tools\SiblingFetcher;
 use BookStack\Http\Controller;
 use Illuminate\Http\Request;
+use Illuminate\Pagination\LengthAwarePaginator;
 
 class SearchController extends Controller
 {
@@ -23,20 +24,21 @@ public function search(Request $request, SearchResultsFormatter $formatter)
     {
         $searchOpts = SearchOptions::fromRequest($request);
         $fullSearchString = $searchOpts->toString();
-        $this->setPageTitle(trans('entities.search_for_term', ['term' => $fullSearchString]));
-
         $page = intval($request->get('page', '0')) ?: 1;
-        $nextPageLink = url('/search?term=' . urlencode($fullSearchString) . '&page=' . ($page + 1));
 
         $results = $this->searchRunner->searchEntities($searchOpts, 'all', $page, 20);
         $formatter->format($results['results']->all(), $searchOpts);
+        $paginator = new LengthAwarePaginator($results['results'], $results['total'], 20, $page);
+        $paginator->setPath('/search');
+        $paginator->appends($request->except('page'));
+
+        $this->setPageTitle(trans('entities.search_for_term', ['term' => $fullSearchString]));
 
         return view('search.all', [
             'entities'     => $results['results'],
             'totalResults' => $results['total'],
+            'paginator'    => $paginator,
             'searchTerm'   => $fullSearchString,
-            'hasNextPage'  => $results['has_more'],
-            'nextPageLink' => $nextPageLink,
             'options'      => $searchOpts,
         ]);
     }
@@ -128,7 +130,7 @@ public function searchSuggestions(Request $request)
     }
 
     /**
-     * Search siblings items in the system.
+     * Search sibling items in the system.
      */
     public function searchSiblings(Request $request, SiblingFetcher $siblingFetcher)
     {