fix: don't fail the grid when relationship resolution exceeds the query value limit

Tables with wide relationships make the API exceed its own 500 value
limit while resolving them for a full page of rows, failing the whole
listRows request and rendering a 400 error page instead of the table.

Retry the page without relationship selects when that happens, then fill
relationships in over chunks of 10 rows. Rows whose chunk still fails
render without relationship data instead of taking down the page.
Tables that load today are unaffected - the fallback only runs after a
'greater than N values' failure.
This commit is contained in:
harsh mahajan committed 2026-07-31 12:35:10 +05:30
1 parent 5c5c83e261
commit 1ee99447d8
4 files changed
+134 -40

No files matched your search

@@ -2,7 +2,7 @@ import { Dependencies, SPREADSHEET_PAGE_LIMIT } from '$lib/constants';
import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/helpers/load';
import type { PageLoad } from './$types';
import { queries, queryParamToMap } from '$lib/components/filters';
import { buildGridQueries, extractSortFromQueries } from '$database/store';
import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
import { getCollectionService } from '$database/(entity)';
export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
@@ -22,6 +22,21 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
const currentSort = extractSortFromQueries(parsedQueries);
const collectionSdk = getCollectionService(params.region, params.project, database.type);
const documentsPage = await loadGridRows(
collection,
(includeRelationships) =>
buildGridQueries(limit, offset, parsedQueries, collection, includeRelationships),
async (queries) => {
const response = await collectionSdk.listDocuments({
databaseId: params.database,
collectionId: params.collection,
queries
});
return { total: response.total, rows: response.documents };
}
);
return {
offset,
limit,
@@ -29,10 +44,6 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
query,
currentSort,
parsedQueries,
documents: await collectionSdk.listDocuments({
databaseId: params.database,
collectionId: params.collection,
queries: buildGridQueries(limit, offset, parsedQueries, collection)
})
documents: { total: documentsPage.total, documents: documentsPage.rows }
};
};
@@ -3,7 +3,8 @@ import type { Column } from '$lib/helpers/types';
import { IconCloudUpload, IconCog } from '@appwrite.io/pink-icons-svelte';
import { resolveRoute, withPath } from '$lib/stores/navigation';
import type { Page } from '@sveltejs/kit';
import { type Models, Query } from '@appwrite.io/console';
import { AppwriteException, type Models, Query } from '@appwrite.io/console';
import { chunks } from '$lib/helpers/array';
import type { Entity, Field } from '$database/(entity)';
import { isRelationship } from '$database/table-[table]/rows/store';
import type { TagValue } from '$lib/components/filters/store';
@@ -178,7 +179,8 @@ export function buildGridQueries(
limit: number,
offset: number,
parsedQueries: Map<TagValue, string>,
table: Entity
table: Entity,
includeRelationships: boolean = true
) {
const hasOrderQuery = Array.from(parsedQueries.values()).some(
(q) => q.includes('orderAsc') || q.includes('orderDesc')
@@ -191,7 +193,75 @@ export function buildGridQueries(
queryArray.push(Query.orderDesc(''));
}
queryArray.push(...parsedQueries.values(), ...buildWildcardEntitiesQuery(table));
queryArray.push(
...parsedQueries.values(),
...(includeRelationships ? buildWildcardEntitiesQuery(table) : [Query.select(['*'])])
);
return queryArray;
}
const RELATIONSHIP_CHUNK_SIZE = 10;
function isTooManyQueryValues(error: unknown): boolean {
return (
error instanceof AppwriteException && /greater than \d+ values/i.test(error.message ?? '')
);
}
type EntityRows<T> = { total: number; rows: T[] };
/**
* Loads a page of the grid, falling back to smaller relationship batches when
* the API trips its own 500 value limit resolving them for the whole page.
*/
export async function loadGridRows<T extends { $id: string }>(
entity: Entity,
buildQueries: (includeRelationships: boolean) => string[],
fetchRows: (queries: string[]) => Promise<EntityRows<T>>
): Promise<EntityRows<T>> {
try {
return await fetchRows(buildQueries(true));
} catch (error) {
if (!isTooManyQueryValues(error)) throw error;
const page = await fetchRows(buildQueries(false));
return {
total: page.total,
rows: await populateRelationships(entity, page.rows, fetchRows)
};
}
}
async function populateRelationships<T extends { $id: string }>(
entity: Entity,
rows: T[],
fetchRows: (queries: string[]) => Promise<EntityRows<T>>
): Promise<T[]> {
const relationshipQueries = buildWildcardEntitiesQuery(entity);
if (relationshipQueries.length <= 1 || rows.length === 0) return rows;
const populated = new Map<string, T>();
await Promise.all(
chunks(rows, RELATIONSHIP_CHUNK_SIZE).map(async (chunk) => {
const rowIds = chunk.map((row) => row.$id);
try {
const response = await fetchRows([
Query.equal('$id', rowIds),
Query.limit(rowIds.length),
...relationshipQueries
]);
response.rows.forEach((row) => populated.set(row.$id, row));
} catch {
// keep the unpopulated rows for this chunk!
}
})
);
return rows.map((row) => populated.get(row.$id) ?? row);
}
@@ -3,7 +3,7 @@ import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/h
import { sdk } from '$lib/stores/sdk';
import type { PageLoad } from './$types';
import { queries, queryParamToMap } from '$lib/components/filters';
import { buildGridQueries, extractSortFromQueries } from '$database/store';
import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
const { table } = await parent();
@@ -28,10 +28,16 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
query,
currentSort,
parsedQueries,
rows: await sdk.forProject(params.region, params.project).tablesDB.listRows({
databaseId: params.database,
tableId: params.table,
queries: buildGridQueries(limit, offset, parsedQueries, table)
})
rows: await loadGridRows(
table,
(includeRelationships) =>
buildGridQueries(limit, offset, parsedQueries, table, includeRelationships),
(queries) =>
sdk.forProject(params.region, params.project).tablesDB.listRows({
databaseId: params.database,
tableId: params.table,
queries
})
)
};
};
@@ -98,6 +98,7 @@
expandTabs,
type Columns,
buildWildcardEntitiesQuery,
loadGridRows,
type SortState,
randomDataModalState,
spreadsheetLoading,
@@ -725,19 +726,22 @@
const filterQueries = parsedQueries.size ? data.parsedQueries.values() : [];
$paginatedRowsLoading = true;
const loadedRows = await sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({
databaseId,
tableId,
queries: [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
...filterQueries /* filter queries */,
...buildWildcardEntitiesQuery(table)
]
});
const loadedRows = await loadGridRows(
table,
(includeRelationships) => [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
...filterQueries /* filter queries */,
...(includeRelationships
? buildWildcardEntitiesQuery(table)
: [Query.select(['*'])])
],
(queries) =>
sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({ databaseId, tableId, queries })
);
paginatedRows.setPage(pageNumber, loadedRows.rows);
$paginatedRowsLoading = false;
@@ -753,18 +757,21 @@
paginatedRows.setMaxPage(targetPageNum);
$paginatedRowsLoading = true;
const loadedRows = await sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({
databaseId,
tableId,
queries: [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
...buildWildcardEntitiesQuery(table)
]
});
const loadedRows = await loadGridRows(
table,
(includeRelationships) => [
getCorrectOrderQuery(),
Query.limit(SPREADSHEET_PAGE_LIMIT),
Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
...(includeRelationships
? buildWildcardEntitiesQuery(table)
: [Query.select(['*'])])
],
(queries) =>
sdk
.forProject(page.params.region, page.params.project)
.tablesDB.listRows({ databaseId, tableId, queries })
);
paginatedRows.setPage(targetPageNum, loadedRows.rows);
$paginatedRowsLoading = false;