diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 23ca49b56..cf3321bcd 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -217,7 +217,7 @@ public function createCollection(string $name, array $attributes = [], array $in \"_createdAt\" TIMESTAMP(3) DEFAULT NULL, \"_updatedAt\" TIMESTAMP(3) DEFAULT NULL, " . \implode(' ', $attributeStrings) . " - _permissions TEXT DEFAULT NULL + _permissions JSONB DEFAULT NULL ); "; @@ -226,20 +226,24 @@ public function createCollection(string $name, array $attributes = [], array $in $createdIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_created"); $updatedIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_updated"); $tenantIdIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_tenant_id"); + $permissionsIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_permissions"); $collection .= " CREATE UNIQUE INDEX \"{$uidIndex}\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai, \"_tenant\"); CREATE INDEX \"{$createdIndex}\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\"); CREATE INDEX \"{$updatedIndex}\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\"); CREATE INDEX \"{$tenantIdIndex}\" ON {$this->getSQLTable($id)} (_tenant, _id); + CREATE INDEX \"{$permissionsIndex}\" ON {$this->getSQLTable($id)} USING gin (_permissions); "; } else { $uidIndex = $this->getShortKey("{$namespace}_{$id}_uid"); $createdIndex = $this->getShortKey("{$namespace}_{$id}_created"); $updatedIndex = $this->getShortKey("{$namespace}_{$id}_updated"); + $permissionsIndex = $this->getShortKey("{$namespace}_{$id}_permissions"); $collection .= " CREATE UNIQUE INDEX \"{$uidIndex}\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai); CREATE INDEX \"{$createdIndex}\" ON {$this->getSQLTable($id)} (\"_createdAt\"); CREATE INDEX \"{$updatedIndex}\" ON {$this->getSQLTable($id)} (\"_updatedAt\"); + CREATE INDEX \"{$permissionsIndex}\" ON {$this->getSQLTable($id)} USING gin (_permissions); "; } @@ -1755,7 +1759,7 @@ protected function getSQLCondition(Query $query, array &$binds, ?string $forColl } /** - * Get vector distance calculation for ORDER BY clause + * Get the SQL expression measuring distance between a vector attribute and the query vector * * @param Query $query * @param array $binds @@ -1763,7 +1767,7 @@ protected function getSQLCondition(Query $query, array &$binds, ?string $forColl * @return string|null * @throws DatabaseException */ - protected function getVectorDistanceOrder(Query $query, array &$binds, string $alias): ?string + protected function getSQLVectorDistance(Query $query, array &$binds, string $alias): ?string { $query->setAttribute($this->getInternalKeyForAttribute($query->getAttribute())); @@ -1785,6 +1789,61 @@ protected function getVectorDistanceOrder(Query $query, array &$binds, string $a }; } + /** + * @param string $distance + * @return string + */ + protected function getSQLReadableDistance(string $distance): string + { + return "{$distance}::text"; + } + + /** + * Match the permission against the copy carried on the row rather than joining the + * permissions table. + * + * Both hold the same fact, written together, but a semi join has to be resolved before + * anything can be ordered, which forces the whole collection to be read whenever the + * ordering could otherwise have come from an index. Matching on the row leaves the + * planner free to cost the permission against the ordering, so a selective permission + * drives from the GIN index and a permissive one is a cheap filter over whichever index + * the ordering wanted. + * + * @param string $collection + * @param array $roles + * @param string $alias + * @param string $type + * @return string + * @throws DatabaseException + */ + protected function getSQLPermissionsCondition( + string $collection, + array $roles, + string $alias, + string $type = Database::PERMISSION_READ + ): string { + if (!\in_array($type, Database::PERMISSIONS)) { + throw new DatabaseException('Unknown permission type: ' . $type); + } + + $column = "{$this->quote($alias)}.{$this->quote('_permissions')}"; + + // Containment rather than jsonb's ?| key operator: PDO reads a lone ? as a positional + // placeholder, and doubling it to escape breaks once a named placeholder is repeated, + // which the cursor conditions do. Each role is its own @> so the index can answer them + // as a BitmapOr; jsonb_exists_any would express it in one call but is not indexable. + $permissions = \array_map( + fn ($role) => "{$column} @> {$this->getPDO()->quote(\json_encode(["{$type}(\"{$role}\")"]))}::jsonb", + $roles + ); + + if ($permissions === []) { + return 'FALSE'; + } + + return '(' . \implode(' OR ', $permissions) . ')'; + } + /** * @param string $value * @return string diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 8d8b6c6c0..2b167704a 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -20,6 +20,8 @@ abstract class SQL extends Adapter { + protected const VECTOR_DISTANCE_COLUMN = '_distance'; + protected mixed $pdo; /** @@ -1810,18 +1812,33 @@ abstract protected function getUpsertStatement( ): mixed; /** - * Get vector distance calculation for ORDER BY clause + * Get the SQL expression measuring distance between a vector attribute and the query vector * * @param Query $query * @param array $binds * @param string $alias * @return string|null */ - protected function getVectorDistanceOrder(Query $query, array &$binds, string $alias): ?string + protected function getSQLVectorDistance(Query $query, array &$binds, string $alias): ?string { return null; } + /** + * Render a vector distance expression in a form safe to read back into PHP + * + * A distance is undefined for a zero vector and can overflow for a large one, so the + * expression can evaluate to NaN or infinity. Those cannot survive the trip into a PHP + * float, so the value is carried as text and interpreted during hydration. + * + * @param string $distance + * @return string + */ + protected function getSQLReadableDistance(string $distance): string + { + return $distance; + } + /** * @param string $value * @return string @@ -3064,18 +3081,17 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : ''; - // Add vector distance calculations to ORDER BY - $vectorOrders = []; + $vectorDistances = []; foreach ($vectorQueries as $query) { - $vectorOrder = $this->getVectorDistanceOrder($query, $binds, $alias); - if ($vectorOrder) { - $vectorOrders[] = $vectorOrder; + $vectorDistance = $this->getSQLVectorDistance($query, $binds, $alias); + if ($vectorDistance) { + $vectorDistances[] = $vectorDistance; } } - if (!empty($vectorOrders)) { + if (!empty($vectorDistances)) { // Vector orders should come first for similarity search - $orders = \array_merge($vectorOrders, $orders); + $orders = \array_merge($vectorDistances, $orders); } $sqlOrder = !empty($orders) ? 'ORDER BY ' . implode(', ', $orders) : ''; @@ -3093,8 +3109,15 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $selections = $this->getAttributeSelections($queries); + $projection = $this->getAttributeProjection($selections, $alias); + + if (!empty($vectorDistances)) { + $readable = $this->getSQLReadableDistance($vectorDistances[0]); + $projection .= ", {$readable} AS {$this->quote(static::VECTOR_DISTANCE_COLUMN)}"; + } + $sql = " - SELECT {$this->getAttributeProjection($selections, $alias)} + SELECT {$projection} FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} {$sqlWhere} {$sqlOrder} @@ -3147,6 +3170,11 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $results[$index]['$permissions'] = \json_decode($document['_permissions'] ?? '[]', true); unset($results[$index]['_permissions']); } + if (\array_key_exists(static::VECTOR_DISTANCE_COLUMN, $document)) { + $value = $document[static::VECTOR_DISTANCE_COLUMN]; + $results[$index][Database::VECTOR_DISTANCE] = \is_numeric($value) ? (float)$value : null; + unset($results[$index][static::VECTOR_DISTANCE_COLUMN]); + } $results[$index] = new Document($results[$index]); } diff --git a/src/Database/Database.php b/src/Database/Database.php index ec93ffe12..69cabf100 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -63,6 +63,9 @@ class Database // Vector types public const VAR_VECTOR = 'vector'; + // Vector query result key + public const VECTOR_DISTANCE = '$distance'; + // Relationship Types public const VAR_RELATIONSHIP = 'relationship'; @@ -8559,7 +8562,19 @@ public function find(string $collection, array $queries = [], string $forPermiss } } - if ($uniqueOrderBy === false) { + $vectorSearch = false; + foreach ($filters as $filter) { + if (\in_array($filter->getMethod(), Query::VECTOR_TYPES)) { + $vectorSearch = true; + break; + } + } + + // A vector search is ordered by distance, and a vector index can only answer that one + // sort key. Appending a tie break makes the ordering unsatisfiable from the index and + // costs a full scan of the collection. The tie break exists to hold a page boundary + // still, so it is only owed to a cursor. + if ($uniqueOrderBy === false && (!$vectorSearch || !empty($cursor))) { $orderAttributes[] = '$sequence'; } diff --git a/tests/e2e/Adapter/PostgresTest.php b/tests/e2e/Adapter/PostgresTest.php index 58beaf64e..14afc6db7 100644 --- a/tests/e2e/Adapter/PostgresTest.php +++ b/tests/e2e/Adapter/PostgresTest.php @@ -7,7 +7,11 @@ use Utopia\Cache\Cache; use Utopia\Database\Adapter\Postgres; use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; use Utopia\Database\PDO; +use Utopia\Database\Query; class PostgresTest extends Base { @@ -72,4 +76,120 @@ protected function deleteIndex(string $collection, string $index): bool return true; } + /** + * Reading must be answerable from the row alone. The permissions table holds the same fact, + * but reaching it needs a join, and a join has to be resolved before anything can be ordered, + * which costs a full read of the collection whenever the ordering could have come from an + * index instead. + */ + public function testReadDoesNotTouchThePermissionsTable(): void + { + $database = $this->getDatabase(); + + // no collection level read, so the permission is enforced per document + $database->createCollection('permsPlan', permissions: [ + Permission::create(Role::any()), + ], documentSecurity: true); + + $database->createAttribute('permsPlan', 'title', Database::VAR_STRING, 64, true); + + foreach (['visible' => Role::any(), 'hidden' => Role::user('nobody')] as $title => $role) { + $database->createDocument('permsPlan', new Document([ + '$permissions' => [Permission::read($role)], + 'title' => $title, + ])); + } + + $table = $database->getNamespace() . '_permsPlan_perms'; + + $scans = function () use ($table): int { + self::$pdo->query('SELECT pg_stat_force_next_flush()'); + self::$pdo->query('SELECT pg_stat_clear_snapshot()'); + + $statement = self::$pdo->prepare('SELECT COALESCE(SUM(seq_scan + COALESCE(idx_scan, 0)), 0) FROM pg_stat_user_tables WHERE relname = :table'); + $statement->execute([':table' => $table]); + + return (int)$statement->fetchColumn(); + }; + + $before = $scans(); + + $results = $database->find('permsPlan'); + + $this->assertCount(1, $results, 'Only the readable document may come back'); + $this->assertSame('visible', $results[0]->getAttribute('title')); + + $this->assertSame( + $before, + $scans(), + 'A read must be satisfied from the row, without reaching the permissions table' + ); + + $database->deleteCollection('permsPlan'); + } + + /** + * A vector search must order by distance alone, because a vector index can answer exactly + * one sort key. Adding a second one does not merely make the index look expensive, it makes + * it unusable, and the collection is read in full instead. + * + * Sequential scans are priced out of the session so that the planner falls back to one only + * when the index genuinely cannot answer the ordering. That separates a hard block from a + * costing preference, and keeps the assertion independent of how many rows are present. + */ + public function testVectorSearchUsesTheIndex(): void + { + $database = $this->getDatabase(); + + $database->createCollection('vectorPlan', permissions: [ + Permission::read(Role::any()), + Permission::create(Role::any()), + ], documentSecurity: false); + + $database->createAttribute('vectorPlan', 'embedding', Database::VAR_VECTOR, 3, true); + $database->createIndex('vectorPlan', 'idx_cosine', Database::INDEX_HNSW_COSINE, ['embedding']); + + for ($i = 0; $i < 50; $i++) { + $database->createDocument('vectorPlan', new Document([ + '$permissions' => [Permission::read(Role::any())], + 'embedding' => [$i / 50, 1 - ($i / 50), 0.0], + ])); + } + + $index = $database->getNamespace() . '_' . $database->getTenant() . '_vectorPlan_idx_cosine'; + + $scans = function () use ($index): int { + self::$pdo->query('SELECT pg_stat_force_next_flush()'); + self::$pdo->query('SELECT pg_stat_clear_snapshot()'); + + $statement = self::$pdo->prepare('SELECT COALESCE(SUM(idx_scan), 0) FROM pg_stat_user_indexes WHERE indexrelname = :index'); + $statement->execute([':index' => $index]); + + return (int)$statement->fetchColumn(); + }; + + $before = $scans(); + + self::$pdo->exec('SET enable_seqscan = off'); + + try { + $results = $database->find('vectorPlan', [ + Query::vectorCosine('embedding', [1.0, 0.0, 0.0]), + Query::limit(10), + ]); + } finally { + self::$pdo->exec('RESET enable_seqscan'); + } + + $this->assertCount(10, $results); + $this->assertEqualsWithDelta(0.0, $results[0]->getAttribute(Database::VECTOR_DISTANCE), 0.001); + + $this->assertGreaterThan( + $before, + $scans(), + 'A vector search must be answerable from the vector index, not by reading the collection' + ); + + $database->deleteCollection('vectorPlan'); + } } diff --git a/tests/e2e/Adapter/Scopes/VectorTests.php b/tests/e2e/Adapter/Scopes/VectorTests.php index 8d84de940..3d229c3e1 100644 --- a/tests/e2e/Adapter/Scopes/VectorTests.php +++ b/tests/e2e/Adapter/Scopes/VectorTests.php @@ -302,6 +302,162 @@ public function testVectorQueries(): void $database->deleteCollection('vectorQueries'); } + public function testVectorDistance(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportForVectors()) { + $this->expectNotToPerformAssertions(); + return; + } + + $database->createCollection('vectorDistance'); + $database->createAttribute('vectorDistance', 'name', Database::VAR_STRING, 255, true); + $database->createAttribute('vectorDistance', 'embedding', Database::VAR_VECTOR, 3, true); + + $vectors = [ + 'identical' => [1.0, 0.0, 0.0], + 'scaled' => [2.0, 0.0, 0.0], + 'orthogonal' => [0.0, 1.0, 0.0], + 'opposite' => [-1.0, 0.0, 0.0], + ]; + + foreach ($vectors as $name => $embedding) { + $database->createDocument('vectorDistance', new Document([ + '$permissions' => [ + Permission::read(Role::any()) + ], + 'name' => $name, + 'embedding' => $embedding, + ])); + } + + $target = [1.0, 0.0, 0.0]; + + $results = $database->find('vectorDistance', [ + Query::vectorCosine('embedding', $target) + ]); + + $this->assertCount(4, $results); + + $cosine = []; + foreach ($results as $result) { + $distance = $result->getAttribute(Database::VECTOR_DISTANCE); + + $this->assertIsFloat($distance, "Cosine distance for '{$result->getAttribute('name')}' must be a float"); + + $cosine[$result->getAttribute('name')] = $distance; + } + + // Cosine distance is 1 - cosine similarity, so magnitude is irrelevant + $this->assertEqualsWithDelta(0.0, $cosine['identical'], 0.000001, 'Identical vector must have zero cosine distance'); + $this->assertEqualsWithDelta(0.0, $cosine['scaled'], 0.000001, 'Cosine distance must ignore magnitude'); + $this->assertEqualsWithDelta(1.0, $cosine['orthogonal'], 0.000001, 'Orthogonal vector must have cosine distance of 1'); + $this->assertEqualsWithDelta(2.0, $cosine['opposite'], 0.000001, 'Opposite vector must have cosine distance of 2'); + + // The returned distance must agree with the order the rows came back in, + // otherwise a caller ranking by the number would disagree with the database + $distances = \array_map( + fn (Document $result) => $result->getAttribute(Database::VECTOR_DISTANCE), + $results + ); + + $sorted = $distances; + \sort($sorted); + $this->assertSame($sorted, $distances, 'Results must be returned in ascending distance order'); + + // Cosine similarity, which is what a caller displays as a relevance score + $this->assertEqualsWithDelta(1.0, 1 - $cosine['identical'], 0.000001); + $this->assertEqualsWithDelta(0.0, 1 - $cosine['orthogonal'], 0.000001); + $this->assertEqualsWithDelta(-1.0, 1 - $cosine['opposite'], 0.000001); + + $results = $database->find('vectorDistance', [ + Query::vectorEuclidean('embedding', $target) + ]); + + $euclidean = []; + foreach ($results as $result) { + $euclidean[$result->getAttribute('name')] = $result->getAttribute(Database::VECTOR_DISTANCE); + } + + $this->assertEqualsWithDelta(0.0, $euclidean['identical'], 0.000001, 'Identical vector must have zero euclidean distance'); + $this->assertEqualsWithDelta(1.0, $euclidean['scaled'], 0.000001, 'Euclidean distance must account for magnitude'); + $this->assertEqualsWithDelta(\sqrt(2), $euclidean['orthogonal'], 0.000001); + $this->assertEqualsWithDelta(2.0, $euclidean['opposite'], 0.000001); + + // The dot product operator returns the negative inner product so that + // ascending order still means most similar first + $results = $database->find('vectorDistance', [ + Query::vectorDot('embedding', $target) + ]); + + $dot = []; + foreach ($results as $result) { + $dot[$result->getAttribute('name')] = $result->getAttribute(Database::VECTOR_DISTANCE); + } + + $this->assertEqualsWithDelta(-1.0, $dot['identical'], 0.000001); + $this->assertEqualsWithDelta(-2.0, $dot['scaled'], 0.000001); + $this->assertEqualsWithDelta(0.0, $dot['orthogonal'], 0.000001); + $this->assertEqualsWithDelta(1.0, $dot['opposite'], 0.000001); + + // A distance is only meaningful relative to a query vector, so a plain + // find must not carry one + $results = $database->find('vectorDistance'); + + $this->assertCount(4, $results); + foreach ($results as $result) { + $this->assertNull( + $result->getAttribute(Database::VECTOR_DISTANCE), + 'A find without a vector query must not return a distance' + ); + } + + // Selecting a subset of attributes builds a different projection + $results = $database->find('vectorDistance', [ + Query::select(['name']), + Query::vectorCosine('embedding', $target), + Query::limit(1), + ]); + + $this->assertCount(1, $results); + $this->assertSame('identical', $results[0]->getAttribute('name')); + $this->assertEqualsWithDelta(0.0, $results[0]->getAttribute(Database::VECTOR_DISTANCE), 0.000001); + + // Cosine distance to a zero vector divides by a zero magnitude, so the engine answers + // NaN. That has no honest float representation and must not read back as 0.0, which + // would claim the pair is identical + $database->createDocument('vectorDistance', new Document([ + '$permissions' => [ + Permission::read(Role::any()) + ], + 'name' => 'zero', + 'embedding' => [0.0, 0.0, 0.0], + ])); + + $results = $database->find('vectorDistance', [ + Query::vectorCosine('embedding', $target) + ]); + + $this->assertCount(5, $results); + + $zero = null; + foreach ($results as $result) { + if ($result->getAttribute('name') === 'zero') { + $zero = $result; + } + } + + $this->assertNotNull($zero, 'The zero vector must still be returned'); + $this->assertNull( + $zero->getAttribute(Database::VECTOR_DISTANCE), + 'An undefined distance must be null, not a number' + ); + + $database->deleteCollection('vectorDistance'); + } + public function testVectorQueryValidation(): void { /** @var Database $database */