mirror of
https://github.com/appwrite/console.git
synced 2026-11-18 20:47:21 +00:00
Merge pull request #3143 from appwrite/fix-grid-relationship-query-limit
fix: don't fail the grid when relationship resolution exceeds the query value limit
This commit is contained in:
4 files changed
+136
-41
No files matched your search
+17
-6
@@ -2,7 +2,7 @@ import { Dependencies, SPREADSHEET_PAGE_LIMIT } from '$lib/constants';
|
||||
import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/helpers/load';
|
||||
import type { PageLoad } from './$types';
|
||||
import { queries, queryParamToMap } from '$lib/components/filters';
|
||||
import { buildGridQueries, extractSortFromQueries } from '$database/store';
|
||||
import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
|
||||
import { getCollectionService } from '$database/(entity)';
|
||||
|
||||
export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
|
||||
@@ -22,6 +22,21 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
|
||||
const currentSort = extractSortFromQueries(parsedQueries);
|
||||
const collectionSdk = getCollectionService(params.region, params.project, database.type);
|
||||
|
||||
const documentsPage = await loadGridRows(
|
||||
collection,
|
||||
(includeRelationships) =>
|
||||
buildGridQueries(limit, offset, parsedQueries, collection, includeRelationships),
|
||||
async (queries) => {
|
||||
const response = await collectionSdk.listDocuments({
|
||||
databaseId: params.database,
|
||||
collectionId: params.collection,
|
||||
queries
|
||||
});
|
||||
|
||||
return { total: response.total, rows: response.documents };
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
offset,
|
||||
limit,
|
||||
@@ -29,10 +44,6 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
|
||||
query,
|
||||
currentSort,
|
||||
parsedQueries,
|
||||
documents: await collectionSdk.listDocuments({
|
||||
databaseId: params.database,
|
||||
collectionId: params.collection,
|
||||
queries: buildGridQueries(limit, offset, parsedQueries, collection)
|
||||
})
|
||||
documents: { total: documentsPage.total, documents: documentsPage.rows }
|
||||
};
|
||||
};
|
||||
+73
-3
@@ -3,7 +3,8 @@ import type { Column } from '$lib/helpers/types';
|
||||
import { IconCloudUpload, IconCog } from '@appwrite.io/pink-icons-svelte';
|
||||
import { resolveRoute, withPath } from '$lib/stores/navigation';
|
||||
import type { Page } from '@sveltejs/kit';
|
||||
import { type Models, Query } from '@appwrite.io/console';
|
||||
import { AppwriteException, type Models, Query } from '@appwrite.io/console';
|
||||
import { chunks } from '$lib/helpers/array';
|
||||
import type { Entity, Field } from '$database/(entity)';
|
||||
import { isRelationship } from '$database/table-[table]/rows/store';
|
||||
import type { TagValue } from '$lib/components/filters/store';
|
||||
@@ -178,7 +179,8 @@ export function buildGridQueries(
|
||||
limit: number,
|
||||
offset: number,
|
||||
parsedQueries: Map<TagValue, string>,
|
||||
table: Entity
|
||||
table: Entity,
|
||||
includeRelationships: boolean = true
|
||||
) {
|
||||
const hasOrderQuery = Array.from(parsedQueries.values()).some(
|
||||
(q) => q.includes('orderAsc') || q.includes('orderDesc')
|
||||
@@ -191,7 +193,75 @@ export function buildGridQueries(
|
||||
queryArray.push(Query.orderDesc(''));
|
||||
}
|
||||
|
||||
queryArray.push(...parsedQueries.values(), ...buildWildcardEntitiesQuery(table));
|
||||
queryArray.push(
|
||||
...parsedQueries.values(),
|
||||
...(includeRelationships ? buildWildcardEntitiesQuery(table) : [Query.select(['*'])])
|
||||
);
|
||||
|
||||
return queryArray;
|
||||
}
|
||||
|
||||
const RELATIONSHIP_CHUNK_SIZE = 10;
|
||||
|
||||
function isTooManyQueryValues(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof AppwriteException && /greater than \d+ values/i.test(error.message ?? '')
|
||||
);
|
||||
}
|
||||
|
||||
type EntityRows<T> = { total: number; rows: T[] };
|
||||
|
||||
/**
|
||||
* Loads a page of the grid, falling back to smaller relationship batches when
|
||||
* the API trips its own 500 value limit resolving them for the whole page.
|
||||
*/
|
||||
export async function loadGridRows<T extends { $id: string }>(
|
||||
entity: Entity,
|
||||
buildQueries: (includeRelationships: boolean) => string[],
|
||||
fetchRows: (queries: string[]) => Promise<EntityRows<T>>
|
||||
): Promise<EntityRows<T>> {
|
||||
try {
|
||||
return await fetchRows(buildQueries(true));
|
||||
} catch (error) {
|
||||
if (!isTooManyQueryValues(error)) throw error;
|
||||
|
||||
const page = await fetchRows(buildQueries(false));
|
||||
|
||||
return {
|
||||
total: page.total,
|
||||
rows: await populateRelationships(entity, page.rows, fetchRows)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function populateRelationships<T extends { $id: string }>(
|
||||
entity: Entity,
|
||||
rows: T[],
|
||||
fetchRows: (queries: string[]) => Promise<EntityRows<T>>
|
||||
): Promise<T[]> {
|
||||
const relationshipQueries = buildWildcardEntitiesQuery(entity);
|
||||
|
||||
if (relationshipQueries.length <= 1 || rows.length === 0) return rows;
|
||||
|
||||
const populated = new Map<string, T>();
|
||||
|
||||
await Promise.all(
|
||||
chunks(rows, RELATIONSHIP_CHUNK_SIZE).map(async (chunk) => {
|
||||
const rowIds = chunk.map((row) => row.$id);
|
||||
|
||||
try {
|
||||
const response = await fetchRows([
|
||||
Query.equal('$id', rowIds),
|
||||
Query.limit(rowIds.length),
|
||||
...relationshipQueries
|
||||
]);
|
||||
|
||||
response.rows.forEach((row) => populated.set(row.$id, row));
|
||||
} catch {
|
||||
// keep the unpopulated rows for this chunk!
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
return rows.map((row) => populated.get(row.$id) ?? row);
|
||||
}
|
||||
+12
-6
@@ -3,7 +3,7 @@ import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/h
|
||||
import { sdk } from '$lib/stores/sdk';
|
||||
import type { PageLoad } from './$types';
|
||||
import { queries, queryParamToMap } from '$lib/components/filters';
|
||||
import { buildGridQueries, extractSortFromQueries } from '$database/store';
|
||||
import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store';
|
||||
|
||||
export const load: PageLoad = async ({ params, depends, url, route, parent }) => {
|
||||
const { table } = await parent();
|
||||
@@ -28,10 +28,16 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) =>
|
||||
query,
|
||||
currentSort,
|
||||
parsedQueries,
|
||||
rows: await sdk.forProject(params.region, params.project).tablesDB.listRows({
|
||||
databaseId: params.database,
|
||||
tableId: params.table,
|
||||
queries: buildGridQueries(limit, offset, parsedQueries, table)
|
||||
})
|
||||
rows: await loadGridRows(
|
||||
table,
|
||||
(includeRelationships) =>
|
||||
buildGridQueries(limit, offset, parsedQueries, table, includeRelationships),
|
||||
(queries) =>
|
||||
sdk.forProject(params.region, params.project).tablesDB.listRows({
|
||||
databaseId: params.database,
|
||||
tableId: params.table,
|
||||
queries
|
||||
})
|
||||
)
|
||||
};
|
||||
};
|
||||
+34
-26
@@ -98,6 +98,7 @@
|
||||
expandTabs,
|
||||
type Columns,
|
||||
buildWildcardEntitiesQuery,
|
||||
loadGridRows,
|
||||
type SortState,
|
||||
randomDataModalState,
|
||||
spreadsheetLoading,
|
||||
@@ -722,22 +723,26 @@
|
||||
}
|
||||
|
||||
const parsedQueries = data.parsedQueries;
|
||||
const filterQueries = parsedQueries.size ? data.parsedQueries.values() : [];
|
||||
// materialized: the fallback in `loadGridRows` builds the queries more than once.
|
||||
const filterQueries = parsedQueries.size ? Array.from(data.parsedQueries.values()) : [];
|
||||
|
||||
$paginatedRowsLoading = true;
|
||||
const loadedRows = await sdk
|
||||
.forProject(page.params.region, page.params.project)
|
||||
.tablesDB.listRows({
|
||||
databaseId,
|
||||
tableId,
|
||||
queries: [
|
||||
getCorrectOrderQuery(),
|
||||
Query.limit(SPREADSHEET_PAGE_LIMIT),
|
||||
Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
|
||||
...filterQueries /* filter queries */,
|
||||
...buildWildcardEntitiesQuery(table)
|
||||
]
|
||||
});
|
||||
const loadedRows = await loadGridRows(
|
||||
table,
|
||||
(includeRelationships) => [
|
||||
getCorrectOrderQuery(),
|
||||
Query.limit(SPREADSHEET_PAGE_LIMIT),
|
||||
Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)),
|
||||
...filterQueries /* filter queries */,
|
||||
...(includeRelationships
|
||||
? buildWildcardEntitiesQuery(table)
|
||||
: [Query.select(['*'])])
|
||||
],
|
||||
(queries) =>
|
||||
sdk
|
||||
.forProject(page.params.region, page.params.project)
|
||||
.tablesDB.listRows({ databaseId, tableId, queries })
|
||||
);
|
||||
|
||||
paginatedRows.setPage(pageNumber, loadedRows.rows);
|
||||
$paginatedRowsLoading = false;
|
||||
@@ -753,18 +758,21 @@
|
||||
paginatedRows.setMaxPage(targetPageNum);
|
||||
$paginatedRowsLoading = true;
|
||||
|
||||
const loadedRows = await sdk
|
||||
.forProject(page.params.region, page.params.project)
|
||||
.tablesDB.listRows({
|
||||
databaseId,
|
||||
tableId,
|
||||
queries: [
|
||||
getCorrectOrderQuery(),
|
||||
Query.limit(SPREADSHEET_PAGE_LIMIT),
|
||||
Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
|
||||
...buildWildcardEntitiesQuery(table)
|
||||
]
|
||||
});
|
||||
const loadedRows = await loadGridRows(
|
||||
table,
|
||||
(includeRelationships) => [
|
||||
getCorrectOrderQuery(),
|
||||
Query.limit(SPREADSHEET_PAGE_LIMIT),
|
||||
Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)),
|
||||
...(includeRelationships
|
||||
? buildWildcardEntitiesQuery(table)
|
||||
: [Query.select(['*'])])
|
||||
],
|
||||
(queries) =>
|
||||
sdk
|
||||
.forProject(page.params.region, page.params.project)
|
||||
.tablesDB.listRows({ databaseId, tableId, queries })
|
||||
);
|
||||
|
||||
paginatedRows.setPage(targetPageNum, loadedRows.rows);
|
||||
$paginatedRowsLoading = false;
|
||||
|
||||
Reference in new issue
Block a user