Merge pull request #3143 from appwrite/fix-grid-relationship-query-limit

fix: don't fail the grid when relationship resolution exceeds the query value limit
This commit is contained in:
Harsh Mahajan authored and GitHub committed 2026-07-31 12:58:36 +05:30
commit 826bccc7f4
4 files changed
+136 -41

No files matched your search

@@ -2,7 +2,7 @@ import { Dependencies, SPREADSHEET_PAGE_LIMIT } from '$lib/constants';
import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/helpers/load';
import type { PageLoad } from './$types';
import { queries, queryParamToMap } from '$lib/components/filters';
import { buildGridQueries, extractSortFromQueries } from '$database/store';
import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
import { getCollectionService } from '$database/(entity)';
export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
@@ -22,6 +22,21 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
const currentSort = extractSortFromQueries(parsedQueries);
const collectionSdk = getCollectionService(params.region, params.project, database.type);
const documentsPage = await loadGridRows(
collection,
(includeRelationships) =>
buildGridQueries(limit, offset, parsedQueries, collection, includeRelationships),
async (queries) => {
const response = await collectionSdk.listDocuments({
databaseId: params.database,
collectionId: params.collection,
queries
});
return { total: response.total, rows: response.documents };
}
);
return {
offset,
limit,
@@ -29,10 +44,6 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
query,
currentSort,
parsedQueries,
documents: await collectionSdk.listDocuments({
databaseId: params.database,
collectionId: params.collection,
queries: buildGridQueries(limit, offset, parsedQueries, collection)
})
documents: { total: documentsPage.total, documents: documentsPage.rows }
};
};
@@ -3,7 +3,8 @@ import type { Column } from '$lib/helpers/types';
import { IconCloudUpload, IconCog } from '@appwrite.io/pink-icons-svelte';
import { resolveRoute, withPath } from '$lib/stores/navigation';
import type { Page } from '@sveltejs/kit';
import { type Models, Query } from '@appwrite.io/console';
import { AppwriteException, type Models, Query } from '@appwrite.io/console';
import { chunks } from '$lib/helpers/array';
import type { Entity, Field } from '$database/(entity)';
import { isRelationship } from '$database/table-[table]/rows/store';
import type { TagValue } from '$lib/components/filters/store';
@@ -178,7 +179,8 @@ export function buildGridQueries(
limit: number,
offset: number,
parsedQueries: Map<TagValue, string>,
table: Entity
table: Entity,
includeRelationships: boolean = true
) {
const hasOrderQuery = Array.from(parsedQueries.values()).some(
(q) => q.includes('orderAsc') || q.includes('orderDesc')
@@ -191,7 +193,75 @@ export function buildGridQueries(
queryArray.push(Query.orderDesc(''));
}
queryArray.push(...parsedQueries.values(), ...buildWildcardEntitiesQuery(table));
queryArray.push(
...parsedQueries.values(),
...(includeRelationships ? buildWildcardEntitiesQuery(table) : [Query.select(['*'])])
);
return queryArray;
}
const RELATIONSHIP_CHUNK_SIZE = 10;
function isTooManyQueryValues(error: unknown): boolean {
return (
error instanceof AppwriteException && /greater than \d+ values/i.test(error.message ?? '')
);
}
type EntityRows<T> = { total: number; rows: T[] };
/**
* Loads a page of the grid, falling back to smaller relationship batches when
* the API trips its own 500 value limit resolving them for the whole page.
*/
export async function loadGridRows<T extends { $id: string }>(
entity: Entity,
buildQueries: (includeRelationships: boolean) => string[],
fetchRows: (queries: string[]) => Promise<EntityRows<T>>
): Promise<EntityRows<T>> {
try {
return await fetchRows(buildQueries(true));
} catch (error) {
if (!isTooManyQueryValues(error)) throw error;
const page = await fetchRows(buildQueries(false));
return {
total: page.total,
rows: await populateRelationships(entity, page.rows, fetchRows)
};
}
}
async function populateRelationships<T extends { $id: string }>(
entity: Entity,
rows: T[],
fetchRows: (queries: string[]) => Promise<EntityRows<T>>
): Promise<T[]> {
const relationshipQueries = buildWildcardEntitiesQuery(entity);
if (relationshipQueries.length <= 1 || rows.length === 0) return rows;
const populated = new Map<string, T>();
await Promise.all(
chunks(rows, RELATIONSHIP_CHUNK_SIZE).map(async (chunk) => {
const rowIds = chunk.map((row) => row.$id);
try {
const response = await fetchRows([
Query.equal('$id', rowIds),
Query.limit(rowIds.length),
...relationshipQueries
]);
response.rows.forEach((row) => populated.set(row.$id, row));
} catch {
// keep the unpopulated rows for this chunk!
}
})
);
return rows.map((row) => populated.get(row.$id) ?? row);
}
@@ -3,7 +3,7 @@ import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/h
import { sdk } from '$lib/stores/sdk';
import type { PageLoad } from './$types';
import { queries, queryParamToMap } from '$lib/components/filters';
import { buildGridQueries, extractSortFromQueries } from '$database/store';
import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
const { table } = await parent();
@@ -28,10 +28,16 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
query,
currentSort,
parsedQueries,
rows: await sdk.forProject(params.region, params.project).tablesDB.listRows({
databaseId: params.database,
tableId: params.table,
queries: buildGridQueries(limit, offset, parsedQueries, table)
})
rows: await loadGridRows(
table,
(includeRelationships) =>
buildGridQueries(limit, offset, parsedQueries, table, includeRelationships),
(queries) =>
sdk.forProject(params.region, params.project).tablesDB.listRows({
databaseId: params.database,
tableId: params.table,
queries
})
)
};
};
@@ -98,6 +98,7 @@
expandTabs,
type Columns,
buildWildcardEntitiesQuery,
loadGridRows,
type SortState,
randomDataModalState,
spreadsheetLoading,
@@ -722,22 +723,26 @@
}
const parsedQueries = data.parsedQueries;
const filterQueries = parsedQueries.size ? data.parsedQueries.values() : [];
// materialized: the fallback in `loadGridRows` builds the queries more than once.
const filterQueries = parsedQueries.size ? Array.from(data.parsedQueries.values()) : [];
$paginatedRowsLoading = true;
const loadedRows = await sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({
databaseId,
tableId,
queries: [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
...filterQueries /* filter queries */,
...buildWildcardEntitiesQuery(table)
]
});
const loadedRows = await loadGridRows(
table,
(includeRelationships) => [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
...filterQueries /* filter queries */,
...(includeRelationships
? buildWildcardEntitiesQuery(table)
: [Query.select(['*'])])
],
(queries) =>
sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({ databaseId, tableId, queries })
);
paginatedRows.setPage(pageNumber, loadedRows.rows);
$paginatedRowsLoading = false;
@@ -753,18 +758,21 @@
paginatedRows.setMaxPage(targetPageNum);
$paginatedRowsLoading = true;
const loadedRows = await sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({
databaseId,
tableId,
queries: [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
...buildWildcardEntitiesQuery(table)
]
});
const loadedRows = await loadGridRows(
table,
(includeRelationships) => [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
...(includeRelationships
? buildWildcardEntitiesQuery(table)
: [Query.select(['*'])])
],
(queries) =>
sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({ databaseId, tableId, queries })
);
paginatedRows.setPage(targetPageNum, loadedRows.rows);
$paginatedRowsLoading = false;