Merge pull request #929 from utopia-php/feat/vector-distance

feat: return the distance a vector query ranked by, and let the index answer it
This commit is contained in:
Jake Barnby authored and GitHub committed 2026-08-04 14:09:58 +12:00
commit 69e6cb5078
5 files changed
+392 -14

No files matched your search

+62 -3
View File
@@ -217,7 +217,7 @@ class Postgres extends SQL
\"_createdAt\" TIMESTAMP(3) DEFAULT NULL,
\"_updatedAt\" TIMESTAMP(3) DEFAULT NULL,
" . \implode(' ', $attributeStrings) . "
_permissions TEXT DEFAULT NULL
_permissions JSONB DEFAULT NULL
);
";
@@ -226,20 +226,24 @@ class Postgres extends SQL
$createdIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_created");
$updatedIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_updated");
$tenantIdIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_tenant_id");
$permissionsIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_permissions");
$collection .= "
CREATE UNIQUE INDEX \"{$uidIndex}\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai, \"_tenant\");
CREATE INDEX \"{$createdIndex}\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\");
CREATE INDEX \"{$updatedIndex}\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\");
CREATE INDEX \"{$tenantIdIndex}\" ON {$this->getSQLTable($id)} (_tenant, _id);
CREATE INDEX \"{$permissionsIndex}\" ON {$this->getSQLTable($id)} USING gin (_permissions);
";
} else {
$uidIndex = $this->getShortKey("{$namespace}_{$id}_uid");
$createdIndex = $this->getShortKey("{$namespace}_{$id}_created");
$updatedIndex = $this->getShortKey("{$namespace}_{$id}_updated");
$permissionsIndex = $this->getShortKey("{$namespace}_{$id}_permissions");
$collection .= "
CREATE UNIQUE INDEX \"{$uidIndex}\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai);
CREATE INDEX \"{$createdIndex}\" ON {$this->getSQLTable($id)} (\"_createdAt\");
CREATE INDEX \"{$updatedIndex}\" ON {$this->getSQLTable($id)} (\"_updatedAt\");
CREATE INDEX \"{$permissionsIndex}\" ON {$this->getSQLTable($id)} USING gin (_permissions);
";
}
@@ -1755,7 +1759,7 @@ class Postgres extends SQL
}
/**
* Get vector distance calculation for ORDER BY clause
* Get the SQL expression measuring distance between a vector attribute and the query vector
*
* @param Query $query
* @param array<string, mixed> $binds
@@ -1763,7 +1767,7 @@ class Postgres extends SQL
* @return string|null
* @throws DatabaseException
*/
protected function getVectorDistanceOrder(Query $query, array &$binds, string $alias): ?string
protected function getSQLVectorDistance(Query $query, array &$binds, string $alias): ?string
{
$query->setAttribute($this->getInternalKeyForAttribute($query->getAttribute()));
@@ -1785,6 +1789,61 @@ class Postgres extends SQL
};
}
/**
* @param string $distance
* @return string
*/
protected function getSQLReadableDistance(string $distance): string
{
return "{$distance}::text";
}
/**
* Match the permission against the copy carried on the row rather than joining the
* permissions table.
*
* Both hold the same fact, written together, but a semi join has to be resolved before
* anything can be ordered, which forces the whole collection to be read whenever the
* ordering could otherwise have come from an index. Matching on the row leaves the
* planner free to cost the permission against the ordering, so a selective permission
* drives from the GIN index and a permissive one is a cheap filter over whichever index
* the ordering wanted.
*
* @param string $collection
* @param array<string> $roles
* @param string $alias
* @param string $type
* @return string
* @throws DatabaseException
*/
protected function getSQLPermissionsCondition(
string $collection,
array $roles,
string $alias,
string $type = Database::PERMISSION_READ
): string {
if (!\in_array($type, Database::PERMISSIONS)) {
throw new DatabaseException('Unknown permission type: ' . $type);
}
$column = "{$this->quote($alias)}.{$this->quote('_permissions')}";
// Containment rather than jsonb's ?| key operator: PDO reads a lone ? as a positional
// placeholder, and doubling it to escape breaks once a named placeholder is repeated,
// which the cursor conditions do. Each role is its own @> so the index can answer them
// as a BitmapOr; jsonb_exists_any would express it in one call but is not indexable.
$permissions = \array_map(
fn ($role) => "{$column} @> {$this->getPDO()->quote(\json_encode(["{$type}(\"{$role}\")"]))}::jsonb",
$roles
);
if ($permissions === []) {
return 'FALSE';
}
return '(' . \implode(' OR ', $permissions) . ')';
}
/**
* @param string $value
* @return string
+38 -10
View File
@@ -20,6 +20,8 @@ use Utopia\Database\Query;
abstract class SQL extends Adapter
{
protected const VECTOR_DISTANCE_COLUMN = '_distance';
protected mixed $pdo;
/**
@@ -1810,18 +1812,33 @@ abstract class SQL extends Adapter
): mixed;
/**
* Get vector distance calculation for ORDER BY clause
* Get the SQL expression measuring distance between a vector attribute and the query vector
*
* @param Query $query
* @param array<string, mixed> $binds
* @param string $alias
* @return string|null
*/
protected function getVectorDistanceOrder(Query $query, array &$binds, string $alias): ?string
protected function getSQLVectorDistance(Query $query, array &$binds, string $alias): ?string
{
return null;
}
/**
* Render a vector distance expression in a form safe to read back into PHP
*
* A distance is undefined for a zero vector and can overflow for a large one, so the
* expression can evaluate to NaN or infinity. Those cannot survive the trip into a PHP
* float, so the value is carried as text and interpreted during hydration.
*
* @param string $distance
* @return string
*/
protected function getSQLReadableDistance(string $distance): string
{
return $distance;
}
/**
* @param string $value
* @return string
@@ -3064,18 +3081,17 @@ abstract class SQL extends Adapter
$sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : '';
// Add vector distance calculations to ORDER BY
$vectorOrders = [];
$vectorDistances = [];
foreach ($vectorQueries as $query) {
$vectorOrder = $this->getVectorDistanceOrder($query, $binds, $alias);
if ($vectorOrder) {
$vectorOrders[] = $vectorOrder;
$vectorDistance = $this->getSQLVectorDistance($query, $binds, $alias);
if ($vectorDistance) {
$vectorDistances[] = $vectorDistance;
}
}
if (!empty($vectorOrders)) {
if (!empty($vectorDistances)) {
// Vector orders should come first for similarity search
$orders = \array_merge($vectorOrders, $orders);
$orders = \array_merge($vectorDistances, $orders);
}
$sqlOrder = !empty($orders) ? 'ORDER BY ' . implode(', ', $orders) : '';
@@ -3093,8 +3109,15 @@ abstract class SQL extends Adapter
$selections = $this->getAttributeSelections($queries);
$projection = $this->getAttributeProjection($selections, $alias);
if (!empty($vectorDistances)) {
$readable = $this->getSQLReadableDistance($vectorDistances[0]);
$projection .= ", {$readable} AS {$this->quote(static::VECTOR_DISTANCE_COLUMN)}";
}
$sql = "
SELECT {$this->getAttributeProjection($selections, $alias)}
SELECT {$projection}
FROM {$this->getSQLTable($name)} AS {$this->quote($alias)}
{$sqlWhere}
{$sqlOrder}
@@ -3147,6 +3170,11 @@ abstract class SQL extends Adapter
$results[$index]['$permissions'] = \json_decode($document['_permissions'] ?? '[]', true);
unset($results[$index]['_permissions']);
}
if (\array_key_exists(static::VECTOR_DISTANCE_COLUMN, $document)) {
$value = $document[static::VECTOR_DISTANCE_COLUMN];
$results[$index][Database::VECTOR_DISTANCE] = \is_numeric($value) ? (float)$value : null;
unset($results[$index][static::VECTOR_DISTANCE_COLUMN]);
}
$results[$index] = new Document($results[$index]);
}
+16 -1
View File
@@ -63,6 +63,9 @@ class Database
// Vector types
public const VAR_VECTOR = 'vector';
// Vector query result key
public const VECTOR_DISTANCE = '$distance';
// Relationship Types
public const VAR_RELATIONSHIP = 'relationship';
@@ -8559,7 +8562,19 @@ class Database
}
}
if ($uniqueOrderBy === false) {
$vectorSearch = false;
foreach ($filters as $filter) {
if (\in_array($filter->getMethod(), Query::VECTOR_TYPES)) {
$vectorSearch = true;
break;
}
}
// A vector search is ordered by distance, and a vector index can only answer that one
// sort key. Appending a tie break makes the ordering unsatisfiable from the index and
// costs a full scan of the collection. The tie break exists to hold a page boundary
// still, so it is only owed to a cursor.
if ($uniqueOrderBy === false && (!$vectorSearch || !empty($cursor))) {
$orderAttributes[] = '$sequence';
}
+120
View File
@@ -7,7 +7,11 @@ use Utopia\Cache\Adapter\Redis as RedisAdapter;
use Utopia\Cache\Cache;
use Utopia\Database\Adapter\Postgres;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\PDO;
use Utopia\Database\Query;
class PostgresTest extends Base
{
@@ -72,4 +76,120 @@ class PostgresTest extends Base
return true;
}
/**
* Reading must be answerable from the row alone. The permissions table holds the same fact,
* but reaching it needs a join, and a join has to be resolved before anything can be ordered,
* which costs a full read of the collection whenever the ordering could have come from an
* index instead.
*/
public function testReadDoesNotTouchThePermissionsTable(): void
{
$database = $this->getDatabase();
// no collection level read, so the permission is enforced per document
$database->createCollection('permsPlan', permissions: [
Permission::create(Role::any()),
], documentSecurity: true);
$database->createAttribute('permsPlan', 'title', Database::VAR_STRING, 64, true);
foreach (['visible' => Role::any(), 'hidden' => Role::user('nobody')] as $title => $role) {
$database->createDocument('permsPlan', new Document([
'$permissions' => [Permission::read($role)],
'title' => $title,
]));
}
$table = $database->getNamespace() . '_permsPlan_perms';
$scans = function () use ($table): int {
self::$pdo->query('SELECT pg_stat_force_next_flush()');
self::$pdo->query('SELECT pg_stat_clear_snapshot()');
$statement = self::$pdo->prepare('SELECT COALESCE(SUM(seq_scan + COALESCE(idx_scan, 0)), 0) FROM pg_stat_user_tables WHERE relname = :table');
$statement->execute([':table' => $table]);
return (int)$statement->fetchColumn();
};
$before = $scans();
$results = $database->find('permsPlan');
$this->assertCount(1, $results, 'Only the readable document may come back');
$this->assertSame('visible', $results[0]->getAttribute('title'));
$this->assertSame(
$before,
$scans(),
'A read must be satisfied from the row, without reaching the permissions table'
);
$database->deleteCollection('permsPlan');
}
/**
* A vector search must order by distance alone, because a vector index can answer exactly
* one sort key. Adding a second one does not merely make the index look expensive, it makes
* it unusable, and the collection is read in full instead.
*
* Sequential scans are priced out of the session so that the planner falls back to one only
* when the index genuinely cannot answer the ordering. That separates a hard block from a
* costing preference, and keeps the assertion independent of how many rows are present.
*/
public function testVectorSearchUsesTheIndex(): void
{
$database = $this->getDatabase();
$database->createCollection('vectorPlan', permissions: [
Permission::read(Role::any()),
Permission::create(Role::any()),
], documentSecurity: false);
$database->createAttribute('vectorPlan', 'embedding', Database::VAR_VECTOR, 3, true);
$database->createIndex('vectorPlan', 'idx_cosine', Database::INDEX_HNSW_COSINE, ['embedding']);
for ($i = 0; $i < 50; $i++) {
$database->createDocument('vectorPlan', new Document([
'$permissions' => [Permission::read(Role::any())],
'embedding' => [$i / 50, 1 - ($i / 50), 0.0],
]));
}
$index = $database->getNamespace() . '_' . $database->getTenant() . '_vectorPlan_idx_cosine';
$scans = function () use ($index): int {
self::$pdo->query('SELECT pg_stat_force_next_flush()');
self::$pdo->query('SELECT pg_stat_clear_snapshot()');
$statement = self::$pdo->prepare('SELECT COALESCE(SUM(idx_scan), 0) FROM pg_stat_user_indexes WHERE indexrelname = :index');
$statement->execute([':index' => $index]);
return (int)$statement->fetchColumn();
};
$before = $scans();
self::$pdo->exec('SET enable_seqscan = off');
try {
$results = $database->find('vectorPlan', [
Query::vectorCosine('embedding', [1.0, 0.0, 0.0]),
Query::limit(10),
]);
} finally {
self::$pdo->exec('RESET enable_seqscan');
}
$this->assertCount(10, $results);
$this->assertEqualsWithDelta(0.0, $results[0]->getAttribute(Database::VECTOR_DISTANCE), 0.001);
$this->assertGreaterThan(
$before,
$scans(),
'A vector search must be answerable from the vector index, not by reading the collection'
);
$database->deleteCollection('vectorPlan');
}
}
+156
View File
@@ -302,6 +302,162 @@ trait VectorTests
$database->deleteCollection('vectorQueries');
}
public function testVectorDistance(): void
{
/** @var Database $database */
$database = static::getDatabase();
if (!$database->getAdapter()->getSupportForVectors()) {
$this->expectNotToPerformAssertions();
return;
}
$database->createCollection('vectorDistance');
$database->createAttribute('vectorDistance', 'name', Database::VAR_STRING, 255, true);
$database->createAttribute('vectorDistance', 'embedding', Database::VAR_VECTOR, 3, true);
$vectors = [
'identical' => [1.0, 0.0, 0.0],
'scaled' => [2.0, 0.0, 0.0],
'orthogonal' => [0.0, 1.0, 0.0],
'opposite' => [-1.0, 0.0, 0.0],
];
foreach ($vectors as $name => $embedding) {
$database->createDocument('vectorDistance', new Document([
'$permissions' => [
Permission::read(Role::any())
],
'name' => $name,
'embedding' => $embedding,
]));
}
$target = [1.0, 0.0, 0.0];
$results = $database->find('vectorDistance', [
Query::vectorCosine('embedding', $target)
]);
$this->assertCount(4, $results);
$cosine = [];
foreach ($results as $result) {
$distance = $result->getAttribute(Database::VECTOR_DISTANCE);
$this->assertIsFloat($distance, "Cosine distance for '{$result->getAttribute('name')}' must be a float");
$cosine[$result->getAttribute('name')] = $distance;
}
// Cosine distance is 1 - cosine similarity, so magnitude is irrelevant
$this->assertEqualsWithDelta(0.0, $cosine['identical'], 0.000001, 'Identical vector must have zero cosine distance');
$this->assertEqualsWithDelta(0.0, $cosine['scaled'], 0.000001, 'Cosine distance must ignore magnitude');
$this->assertEqualsWithDelta(1.0, $cosine['orthogonal'], 0.000001, 'Orthogonal vector must have cosine distance of 1');
$this->assertEqualsWithDelta(2.0, $cosine['opposite'], 0.000001, 'Opposite vector must have cosine distance of 2');
// The returned distance must agree with the order the rows came back in,
// otherwise a caller ranking by the number would disagree with the database
$distances = \array_map(
fn (Document $result) => $result->getAttribute(Database::VECTOR_DISTANCE),
$results
);
$sorted = $distances;
\sort($sorted);
$this->assertSame($sorted, $distances, 'Results must be returned in ascending distance order');
// Cosine similarity, which is what a caller displays as a relevance score
$this->assertEqualsWithDelta(1.0, 1 - $cosine['identical'], 0.000001);
$this->assertEqualsWithDelta(0.0, 1 - $cosine['orthogonal'], 0.000001);
$this->assertEqualsWithDelta(-1.0, 1 - $cosine['opposite'], 0.000001);
$results = $database->find('vectorDistance', [
Query::vectorEuclidean('embedding', $target)
]);
$euclidean = [];
foreach ($results as $result) {
$euclidean[$result->getAttribute('name')] = $result->getAttribute(Database::VECTOR_DISTANCE);
}
$this->assertEqualsWithDelta(0.0, $euclidean['identical'], 0.000001, 'Identical vector must have zero euclidean distance');
$this->assertEqualsWithDelta(1.0, $euclidean['scaled'], 0.000001, 'Euclidean distance must account for magnitude');
$this->assertEqualsWithDelta(\sqrt(2), $euclidean['orthogonal'], 0.000001);
$this->assertEqualsWithDelta(2.0, $euclidean['opposite'], 0.000001);
// The dot product operator returns the negative inner product so that
// ascending order still means most similar first
$results = $database->find('vectorDistance', [
Query::vectorDot('embedding', $target)
]);
$dot = [];
foreach ($results as $result) {
$dot[$result->getAttribute('name')] = $result->getAttribute(Database::VECTOR_DISTANCE);
}
$this->assertEqualsWithDelta(-1.0, $dot['identical'], 0.000001);
$this->assertEqualsWithDelta(-2.0, $dot['scaled'], 0.000001);
$this->assertEqualsWithDelta(0.0, $dot['orthogonal'], 0.000001);
$this->assertEqualsWithDelta(1.0, $dot['opposite'], 0.000001);
// A distance is only meaningful relative to a query vector, so a plain
// find must not carry one
$results = $database->find('vectorDistance');
$this->assertCount(4, $results);
foreach ($results as $result) {
$this->assertNull(
$result->getAttribute(Database::VECTOR_DISTANCE),
'A find without a vector query must not return a distance'
);
}
// Selecting a subset of attributes builds a different projection
$results = $database->find('vectorDistance', [
Query::select(['name']),
Query::vectorCosine('embedding', $target),
Query::limit(1),
]);
$this->assertCount(1, $results);
$this->assertSame('identical', $results[0]->getAttribute('name'));
$this->assertEqualsWithDelta(0.0, $results[0]->getAttribute(Database::VECTOR_DISTANCE), 0.000001);
// Cosine distance to a zero vector divides by a zero magnitude, so the engine answers
// NaN. That has no honest float representation and must not read back as 0.0, which
// would claim the pair is identical
$database->createDocument('vectorDistance', new Document([
'$permissions' => [
Permission::read(Role::any())
],
'name' => 'zero',
'embedding' => [0.0, 0.0, 0.0],
]));
$results = $database->find('vectorDistance', [
Query::vectorCosine('embedding', $target)
]);
$this->assertCount(5, $results);
$zero = null;
foreach ($results as $result) {
if ($result->getAttribute('name') === 'zero') {
$zero = $result;
}
}
$this->assertNotNull($zero, 'The zero vector must still be returned');
$this->assertNull(
$zero->getAttribute(Database::VECTOR_DISTANCE),
'An undefined distance must be null, not a number'
);
$database->deleteCollection('vectorDistance');
}
public function testVectorQueryValidation(): void
{
/** @var Database $database */