feat explain idx

app/init/models.php

@@ -185,6 +185,7 @@ use Appwrite\Utopia\Response\Model\ProviderRepositoryFramework;
 use Appwrite\Utopia\Response\Model\ProviderRepositoryFrameworkList;
 use Appwrite\Utopia\Response\Model\ProviderRepositoryRuntime;
 use Appwrite\Utopia\Response\Model\ProviderRepositoryRuntimeList;
+use Appwrite\Utopia\Response\Model\QueryPlanEntry;
 use Appwrite\Utopia\Response\Model\Report;
 use Appwrite\Utopia\Response\Model\ResourceToken;
 use Appwrite\Utopia\Response\Model\Row;
@@ -236,6 +237,10 @@ Response::setModel(new Any());
 Response::setModel(new Error());
 Response::setModel(new ErrorDev());
 
+// Diagnostics
+Response::setModel(new QueryPlanEntry());
+Response::setModel(new BaseList('Query Plan', Response::MODEL_QUERY_PLAN, 'queries', Response::MODEL_QUERY_PLAN_ENTRY, paging: false));
+
 // Lists
 Response::setModel(new BaseList('Rows List', Response::MODEL_ROW_LIST, 'rows', Response::MODEL_ROW));
 Response::setModel(new BaseList('Documents List', Response::MODEL_DOCUMENT_LIST, 'documents', Response::MODEL_DOCUMENT));

docs/references/tablesdb/explain-rows.md

@@ -0,0 +1,3 @@
+Get a query plan for a `listRows` call without executing it. Useful for diagnosing slow reads and verifying that the indexes you created on a table are actually being used.
+
+Takes the same parameters as `listRows`. Returns one plan entry per physical query Appwrite would have run — including the per-relationship fetches that `listRows` issues sequentially when your query selects related fields. Internal storage details (the permission companion table, the metadata system table, internal column names) are stripped from the output, so the response only references your tables and `$`-prefixed attributes.

src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Explain/Get.php

@@ -0,0 +1,246 @@
+<?php
+
+namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Explain;
+
+use Appwrite\Extend\Exception;
+use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action;
+use Appwrite\SDK\AuthType;
+use Appwrite\SDK\ContentType;
+use Appwrite\SDK\Method;
+use Appwrite\SDK\Response as SDKResponse;
+use Appwrite\Utopia\Database\Documents\User;
+use Appwrite\Utopia\Response as UtopiaResponse;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Exception\Order as OrderException;
+use Utopia\Database\Exception\Query as QueryException;
+use Utopia\Database\Exception\Timeout;
+use Utopia\Database\Query;
+use Utopia\Database\Validator\Authorization;
+use Utopia\Database\Validator\Query\Cursor;
+use Utopia\Database\Validator\UID;
+use Utopia\Http\Adapter\Swoole\Response as SwooleResponse;
+use Utopia\Validator\ArrayList;
+use Utopia\Validator\Text;
+
+/**
+ * Base implementation for the explain-rows / explain-documents endpoint.
+ *
+ * Mirrors the shape of `XList` (listDocuments / listRows) but instead of
+ * executing the underlying read, captures the vendor-native query plan for
+ * each read Appwrite would have issued. Customer-facing output is sanitized
+ * so internal storage details (the `_perms` companion table, `_metadata`
+ * system table, internal column names) never leak.
+ *
+ * Not registered directly — TablesDB (and any future namespace) subclasses
+ * this and overrides the constructor with the namespace-specific URL + SDK
+ * metadata. See TablesDB\Tables\Rows\Explain\Get.
+ */
+abstract class Get extends Action
+{
+    public static function getName(): string
+    {
+        return 'explainRows';
+    }
+
+    protected function getResponseModel(): string
+    {
+        return UtopiaResponse::MODEL_QUERY_PLAN;
+    }
+
+    public function __construct()
+    {
+        $this
+            ->setHttpMethod(self::HTTP_REQUEST_METHOD_GET)
+            ->setHttpPath('/v1/databases/:databaseId/collections/:collectionId/documents/explain')
+            ->desc('Explain rows query plan')
+            ->groups(['api', 'database'])
+            ->label('scope', 'documents.read')
+            ->label('resourceType', RESOURCE_TYPE_DATABASES)
+            ->label('sdk', new Method(
+                namespace: $this->getSDKNamespace(),
+                group: $this->getSDKGroup(),
+                name: self::getName(),
+                description: '/docs/references/databases/explain-rows.md',
+                auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT],
+                responses: [
+                    new SDKResponse(
+                        code: SwooleResponse::STATUS_CODE_OK,
+                        model: $this->getResponseModel(),
+                    )
+                ],
+                contentType: ContentType::JSON,
+            ))
+            ->param('databaseId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Database ID.', false, ['dbForProject'])
+            ->param('collectionId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Collection ID.', false, ['dbForProject'])
+            ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. Same shape as listRows.', true)
+            ->inject('response')
+            ->inject('dbForProject')
+            ->inject('user')
+            ->inject('getDatabasesDB')
+            ->inject('authorization')
+            ->callback($this->action(...));
+    }
+
+    /**
+     * @param string $databaseId
+     * @param string $collectionId
+     * @param array<string> $queries
+     */
+    public function action(
+        string $databaseId,
+        string $collectionId,
+        array $queries,
+        UtopiaResponse $response,
+        Database $dbForProject,
+        User $user,
+        callable $getDatabasesDB,
+        Authorization $authorization,
+    ): void {
+        $isAPIKey = $user->isApp($authorization->getRoles());
+        $isPrivilegedUser = $user->isPrivileged($authorization->getRoles());
+
+        $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
+        if ($database->isEmpty() || (!$database->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) {
+            throw new Exception(Exception::DATABASE_NOT_FOUND, params: [$databaseId]);
+        }
+
+        $collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId));
+        if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) {
+            throw new Exception($this->getParentNotFoundException(), params: [$collectionId]);
+        }
+
+        try {
+            $queries = Query::parseQueries($queries);
+        } catch (QueryException $e) {
+            throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage());
+        }
+
+        $dbForDatabases = $getDatabasesDB($database);
+
+        $cursor = Query::getCursorQueries($queries, false);
+        $cursor = \reset($cursor);
+
+        if ($cursor !== false) {
+            $validator = new Cursor();
+            if (!$validator->isValid($cursor)) {
+                throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription());
+            }
+
+            $documentId = $cursor->getValue();
+
+            $cursorDocument = $authorization->skip(fn () => $dbForDatabases->getDocument('database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), $documentId));
+
+            if ($cursorDocument->isEmpty()) {
+                $type = ucfirst($this->getContext());
+                throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "$type '{$documentId}' for the 'cursor' value not found.");
+            }
+
+            $cursor->setValue($cursorDocument);
+        }
+
+        $collectionTableId = 'database_' . $database->getSequence() . '_collection_' . $collection->getSequence();
+        $hasSelects = !empty(Query::groupByType($queries)['selections']);
+
+        // Mirror listRows: skip relationship resolution when the caller didn't
+        // ask for related selects, to avoid capturing plans for reads the real
+        // endpoint would not have issued either.
+        $find = $hasSelects
+            ? fn () => $dbForDatabases->find($collectionTableId, $queries)
+            : fn () => $dbForDatabases->skipRelationships(fn () => $dbForDatabases->find($collectionTableId, $queries));
+
+        try {
+            $plan = $dbForDatabases->withExplain($find);
+        } catch (OrderException $e) {
+            $documents = $this->isCollectionsAPI() ? 'documents' : 'rows';
+            $attribute = $this->isCollectionsAPI() ? 'attribute' : 'column';
+            $message = "The order $attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all $documents order $attribute values are non-null.";
+            throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, $message);
+        } catch (QueryException $e) {
+            throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage());
+        } catch (Timeout) {
+            throw new Exception(Exception::DATABASE_TIMEOUT);
+        }
+
+        $translated = $this->translatePlanCollections(
+            $plan->getAttribute('queries', []),
+            $database,
+            $collection,
+            $dbForProject,
+            $authorization,
+        );
+
+        $response->dynamic(new Document([
+            'queries' => $translated,
+        ]), $this->getResponseModel());
+    }
+
+    /**
+     * Walk the captured plans and rewrite internal `database_<seq>_collection_<seq>`
+     * references to the user-facing collection ID. Each captured entry came from
+     * a real find() invocation, so the `context.collection` field always carries
+     * the physical table id; this maps it back to the customer's vocabulary
+     * (and resolves related collections by sequence for relationship fetches).
+     *
+     * @param array<int, array<string, mixed>> $entries
+     * @return array<int, Document>
+     */
+    protected function translatePlanCollections(
+        array $entries,
+        Document $database,
+        Document $collection,
+        Database $dbForProject,
+        Authorization $authorization,
+    ): array {
+        $databaseSequence = $database->getSequence();
+        $databaseCollectionsTable = 'database_' . $databaseSequence;
+        $collectionResolver = $this->buildCollectionResolver($database, $collection, $dbForProject, $authorization);
+
+        $output = [];
+        foreach ($entries as $entry) {
+            $context = $entry['context'] ?? [];
+            $physicalCollection = $context['collection'] ?? null;
+
+            if (\is_string($physicalCollection) && \str_starts_with($physicalCollection, $databaseCollectionsTable . '_collection_')) {
+                $relatedSequence = \substr($physicalCollection, \strlen($databaseCollectionsTable . '_collection_'));
+                $context['collection'] = $collectionResolver($relatedSequence) ?? $physicalCollection;
+            }
+
+            $output[] = new Document([
+                'purpose' => $entry['purpose'] ?? 'find',
+                'context' => $context,
+                'plan'    => $entry['plan'] ?? [],
+            ]);
+        }
+
+        return $output;
+    }
+
+    /**
+     * Returns a closure that maps a collection $sequence to its user-facing id,
+     * memoizing lookups for repeat hits during relationship resolution.
+     */
+    protected function buildCollectionResolver(
+        Document $database,
+        Document $primary,
+        Database $dbForProject,
+        Authorization $authorization,
+    ): callable {
+        $cache = [
+            (string) $primary->getSequence() => $primary->getId(),
+        ];
+        $databaseCollectionsTable = 'database_' . $database->getSequence();
+
+        return function (string $sequence) use (&$cache, $databaseCollectionsTable, $dbForProject, $authorization): ?string {
+            if (\array_key_exists($sequence, $cache)) {
+                return $cache[$sequence];
+            }
+            $related = $authorization->skip(fn () => $dbForProject->findOne($databaseCollectionsTable, [
+                Query::equal('$sequence', [$sequence]),
+            ]));
+            $resolved = $related->isEmpty() ? null : $related->getId();
+            $cache[$sequence] = $resolved;
+            return $resolved;
+        };
+    }
+}

src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Explain/Get.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Explain;
+
+use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Explain\Get as DocumentExplain;
+use Appwrite\SDK\AuthType;
+use Appwrite\SDK\ContentType;
+use Appwrite\SDK\Method;
+use Appwrite\SDK\Response as SDKResponse;
+use Appwrite\Utopia\Response as UtopiaResponse;
+use Utopia\Database\Database;
+use Utopia\Database\Validator\UID;
+use Utopia\Http\Adapter\Swoole\Response as SwooleResponse;
+use Utopia\Validator\ArrayList;
+use Utopia\Validator\Text;
+
+/**
+ * TablesDB skin for the explain endpoint.
+ *
+ * Inherits the full action body from DocumentExplain; only re-declares the
+ * URL, scope, response model, and SDK metadata so the path
+ * `/v1/tablesdb/:databaseId/tables/:tableId/rows/explain` flips the parent's
+ * `setHttpPath()` context flag (which in turn picks the rows-vocabulary
+ * exception messages, SDK group, etc.).
+ */
+class Get extends DocumentExplain
+{
+    public static function getName(): string
+    {
+        return 'explainRows';
+    }
+
+    protected function getResponseModel(): string
+    {
+        return UtopiaResponse::MODEL_QUERY_PLAN;
+    }
+
+    public function __construct()
+    {
+        $this
+            ->setHttpMethod(self::HTTP_REQUEST_METHOD_GET)
+            ->setHttpPath('/v1/tablesdb/:databaseId/tables/:tableId/rows/explain')
+            ->desc('Explain rows query plan')
+            ->groups(['api', 'database'])
+            ->label('scope', 'rows.read')
+            ->label('resourceType', RESOURCE_TYPE_DATABASES)
+            ->label('sdk', new Method(
+                namespace: $this->getSDKNamespace(),
+                group: $this->getSDKGroup(),
+                name: self::getName(),
+                description: '/docs/references/tablesdb/explain-rows.md',
+                auth: [AuthType::ADMIN, AuthType::SESSION, AuthType::KEY, AuthType::JWT],
+                responses: [
+                    new SDKResponse(
+                        code: SwooleResponse::STATUS_CODE_OK,
+                        model: $this->getResponseModel(),
+                    )
+                ],
+                contentType: ContentType::JSON,
+            ))
+            ->param('databaseId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Database ID.', false, ['dbForProject'])
+            ->param('tableId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Table ID.', false, ['dbForProject'])
+            ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. Same shape as listRows.', true)
+            ->inject('response')
+            ->inject('dbForProject')
+            ->inject('user')
+            ->inject('getDatabasesDB')
+            ->inject('authorization')
+            ->callback($this->action(...));
+    }
+}

src/Appwrite/Platform/Modules/Databases/Services/Registry/TablesDB.php

@@ -59,6 +59,7 @@ use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Column\Decreme
 use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Column\Increment as IncrementRowColumn;
 use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Create as CreateRow;
 use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Delete as DeleteRow;
+use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Explain\Get as ExplainRows;
 use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Get as GetRow;
 use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Logs\XList as ListRowLogs;
 use Appwrite\Platform\Modules\Databases\Http\TablesDB\Tables\Rows\Update as UpdateRow;
@@ -221,6 +222,7 @@ class TablesDB extends Base
         $service->addAction(DeleteRow::getName(), new DeleteRow());
         $service->addAction(DeleteRows::getName(), new DeleteRows());
         $service->addAction(ListRows::getName(), new ListRows());
+        $service->addAction(ExplainRows::getName(), new ExplainRows());
         $service->addAction(ListRowLogs::getName(), new ListRowLogs());
         $service->addAction(IncrementRowColumn::getName(), new IncrementRowColumn());
         $service->addAction(DecrementRowColumn::getName(), new DecrementRowColumn());

src/Appwrite/Utopia/Response.php

@@ -69,6 +69,8 @@ class Response extends SwooleResponse
     public const MODEL_PRESENCE_LIST = 'presenceList';
     public const MODEL_ROW = 'row';
     public const MODEL_ROW_LIST = 'rowList';
+    public const MODEL_QUERY_PLAN = 'queryPlan';
+    public const MODEL_QUERY_PLAN_ENTRY = 'queryPlanEntry';
 
     // Database Attributes
     public const MODEL_ATTRIBUTE = 'attribute';

src/Appwrite/Utopia/Response/Model/QueryPlanEntry.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace Appwrite\Utopia\Response\Model;
+
+use Appwrite\Utopia\Response;
+
+/**
+ * One captured query plan from a single physical read issued during the
+ * explained operation. A listRows that resolves relationships produces
+ * multiple entries (one per underlying find()), in execution order.
+ */
+class QueryPlanEntry extends Any
+{
+    public function getName(): string
+    {
+        return 'QueryPlanEntry';
+    }
+
+    public function getType(): string
+    {
+        return Response::MODEL_QUERY_PLAN_ENTRY;
+    }
+
+    public function __construct()
+    {
+        $this
+            ->addRule('purpose', [
+                'type' => self::TYPE_STRING,
+                'description' => 'What this read was issued for. Currently always "find"; future values may include "count" or "sum".',
+                'default' => 'find',
+                'example' => 'find',
+            ])
+            ->addRule('context', [
+                'type' => self::TYPE_JSON,
+                'description' => 'Metadata about which user-facing collection this plan refers to (e.g. {"collection": "movies"} or {"collection": "reviews"} for a relationship fetch).',
+                'default' => new \stdClass(),
+                'example' => ['collection' => 'movies'],
+            ])
+            ->addRule('plan', [
+                'type' => self::TYPE_JSON,
+                'description' => 'Vendor-native query plan. Always carries `engine`, `rowsScanned`, `indexUsed`, `estimatedCost`; may also carry a `tree` field with the raw plan for debugging. Internal storage details (the `_perms` companion table, the `_metadata` system table, internal column names) are stripped before returning.',
+                'default' => new \stdClass(),
+                'example' => [
+                    'engine' => 'sql',
+                    'rowsScanned' => 25,
+                    'indexUsed' => 'idx_status_createdAt',
+                    'estimatedCost' => 4.5,
+                ],
+            ]);
+    }
+}

tests/e2e/Services/TablesDB/TablesDBCustomServerTest.php

@@ -2,11 +2,13 @@
 
 namespace Tests\E2E\Services\TablesDB;
 
+use Tests\E2E\Client;
 use Tests\E2E\Scopes\ApiTablesDB;
 use Tests\E2E\Scopes\ProjectCustom;
 use Tests\E2E\Scopes\Scope;
 use Tests\E2E\Scopes\SideServer;
 use Tests\E2E\Services\Databases\DatabasesBase;
+use Utopia\Database\Query;
 
 class TablesDBCustomServerTest extends Scope
 {
@@ -14,4 +16,52 @@ class TablesDBCustomServerTest extends Scope
     use ProjectCustom;
     use SideServer;
     use ApiTablesDB;
+
+    /**
+     * `explainRows` is registered only on the TablesDB namespace (the legacy
+     * /v1/databases path is deprecated, so we deliberately did not expose
+     * `explainDocuments`). Verifying it here rather than in DatabasesBase keeps
+     * the shared trait from accidentally running this against an endpoint that
+     * doesn't exist.
+     */
+    public function testExplainRows(): void
+    {
+        $data = $this->setupDocuments();
+        $databaseId = $data['databaseId'];
+        $tableId = $data['moviesId'];
+
+        $response = $this->client->call(
+            Client::METHOD_GET,
+            $this->getRecordUrl($databaseId, $tableId) . '/explain',
+            array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()),
+            [
+                'queries' => [
+                    Query::orderAsc('releaseYear')->toString(),
+                    Query::limit(10)->toString(),
+                ],
+            ]
+        );
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+        $this->assertArrayHasKey('queries', $response['body']);
+        $this->assertIsArray($response['body']['queries']);
+        $this->assertNotEmpty($response['body']['queries'], 'must capture at least the main find()');
+
+        $first = $response['body']['queries'][0];
+        $this->assertEquals('find', $first['purpose']);
+        $this->assertArrayHasKey('context', $first);
+        $this->assertArrayHasKey('collection', $first['context']);
+        $this->assertEquals($tableId, $first['context']['collection'], 'physical table id must be translated back to the user-facing table id');
+
+        $this->assertArrayHasKey('plan', $first);
+        $this->assertArrayHasKey('engine', $first['plan']);
+
+        // Sanitizer must have stripped any reference to internal storage tables.
+        $rawPlan = json_encode($first['plan']);
+        $this->assertStringNotContainsString('_perms', $rawPlan, 'permission companion table must be redacted');
+        $this->assertStringNotContainsString('__metadata', $rawPlan, 'metadata system table must be redacted');
+    }
 }