update: realtime progress on csv imports!

This commit is contained in:
Darshan committed 2025-04-17 19:03:34 +05:30
1 parent a35a307d78
commit 014d07951d
11 files changed
+833 -1101

No files matched your search

+1 -1
View File
@@ -21,7 +21,7 @@
"e2e:ui": "playwright test --ui"
},
"dependencies": {
"@appwrite.io/console": "https://pkg.pr.new/appwrite/appwrite/@appwrite.io/console@e2f082e",
"@appwrite.io/console": "https://pkg.pr.new/appwrite/appwrite/@appwrite.io/console@61868a9",
"@appwrite.io/pink-icons": "0.25.0",
"@appwrite.io/pink-icons-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@6b17cb23",
"@appwrite.io/pink-legacy": "^1.0.3",
+678 -898
View File
File diff suppressed because it is too large. Load diff
+2
View File
@@ -149,6 +149,7 @@ export enum Click {
DatabaseIndexDelete = 'click_index_delete',
DatabaseCollectionDelete = 'click_collection_delete',
DatabaseDatabaseDelete = 'click_database_delete',
DatabaseImportCsv = 'click_database_import_csv',
DomainCreateClick = 'click_domain_create',
DomainDeleteClick = 'click_domain_delete',
DomainRetryDomainVerificationClick = 'click_domain_retry_domain_verification',
@@ -267,6 +268,7 @@ export enum Submit {
DatabaseCreate = 'submit_database_create',
DatabaseDelete = 'submit_database_delete',
DatabaseUpdateName = 'submit_database_update_name',
DatabaseImportCsv = 'submit_database_import_csv',
AttributeCreate = 'submit_attribute_create',
AttributeUpdate = 'submit_attribute_update',
AttributeDelete = 'submit_attribute_delete',
+108 -45
View File
@@ -1,34 +1,95 @@
<script lang="ts">
import { type Payload } from '@appwrite.io/console';
import { onMount } from 'svelte';
import { goto } from '$app/navigation';
import { addNotification } from '$lib/stores/notifications';
import { base } from '$app/paths';
import { page } from '$app/state';
import { sdk } from '$lib/stores/sdk';
import { Dependencies } from '$lib/constants';
import { goto, invalidate } from '$app/navigation';
import { getProjectId } from '$lib/helpers/project';
import { Typography } from '@appwrite.io/pink-svelte';
import { page } from '$app/stores';
let importItems: Map<string, string> = new Map<string, string>();
import { writable, type Writable } from 'svelte/store';
import { addNotification } from '$lib/stores/notifications';
import { type Models, type Payload, Query } from '@appwrite.io/console';
type ImportItem = {
status: string;
collection?: string;
};
type ImportItemsMap = Map<string, ImportItem>;
/**
* Keeps a track of the active and ongoing csv migrations.
*
* The structure is as follows -
* `{ migrationId: { status: status, collection: collection } }`
*/
const importItems: Writable<ImportItemsMap> = writable(new Map());
async function showCompletionNotification(databaseId: string, collectionId: string) {
const projectId = page.params.project;
await invalidate(Dependencies.DOCUMENTS);
const url = `${base}/project-${projectId}/databases/database-${databaseId}/collection-${collectionId}`;
function showCompletionNotification(databaseId: string, collectionId: string) {
const projectId = $page.params.project;
addNotification({
type: 'success',
isHtml: true,
message: `Documents import through CSV complete.`,
buttons: [
{
name: 'View imported documents',
method: () => {
goto(
`${base}/project-${projectId}/databases/database-${databaseId}/collection-${collectionId}`
);
}
}
]
message: `CSV import finished successfully.`,
buttons:
collectionId === page.params.collection
? undefined
: [
{
name: 'View documents',
method: () => goto(url)
}
]
});
}
// TODO: @itznotabug, update after the SDK is available
function updateOrAddItem(payload: Payload) {}
async function updateOrAddItem(importData: Payload | Models.Migration) {
if (importData.source.toLowerCase() !== 'csv') return;
const existing = $importItems.get(importData.$id);
if (existing?.status === importData.status) return;
const status = importData.status;
const resourceId = importData.resourceId ?? '';
const [databaseId, collectionId] = resourceId.split(':') ?? [];
let collectionName = existing?.collection ?? null;
console.log([importData.status, text(importData.status)].join(', '));
// fetch if we don't have it
if (!existing && collectionId) {
try {
const collection = await sdk.forProject.databases.getCollection(
databaseId,
collectionId
);
collectionName = collection.name;
} catch (err) {
collectionName = null;
}
}
importItems.update((items) => {
items.set(importData.$id, { status, collection: collectionName });
return items;
});
if (status === 'completed') {
await showCompletionNotification(databaseId, collectionId);
}
}
function clear() {
importItems.update((items) => {
items.clear();
return items;
});
}
function graphSize(status: string): number {
switch (status) {
@@ -57,23 +118,22 @@
}
onMount(() => {
// Mocking data
//showCsvImportBox = true;
// importItems.set(ID.unique(), 'something here')
// importItems.set(ID.unique(), 'something here 2')
// no idea yet on what events to observe
// return sdk.forConsole.client.subscribe('console', (response) => {
// if (!response.channels.includes(`projects.${getProjectId()}`)) return;
//
// // TODO: @itznotabug, update after SDK is available
// if (response.events.includes('*')) {
// updateOrAddItem(response.payload);
// }
// });
sdk.forProject.migrations
.list([Query.equal('source', 'CSV'), Query.equal('status', ['pending', 'processing'])])
.then((migrations) => {
migrations.migrations.forEach(updateOrAddItem);
});
return sdk.forConsole.client.subscribe('console', (response) => {
if (!response.channels.includes(`projects.${getProjectId()}`)) return;
if (response.events.includes('migrations.*')) {
updateOrAddItem(response.payload);
}
});
});
$: isOpen = true;
$: showCsvImportBox = importItems.size > 0;
$: showCsvImportBox = $importItems.size > 0;
</script>
{#if showCsvImportBox}
@@ -82,7 +142,7 @@
<header class="upload-box-header">
<h4 class="upload-box-title">
<Typography.Text variant="m-500">
Importing documents ({importItems.size})
Importing documents ({$importItems.size})
</Typography.Text>
</h4>
<button
@@ -92,17 +152,17 @@
on:click={() => {
isOpen = !isOpen;
}}>
<span class="icon-cheveron-up" aria-hidden="true" />
<span class="icon-cheveron-up" aria-hidden="true"></span>
</button>
<button
class="upload-box-button"
aria-label="close backup restore box"
on:click={() => (importItems = new Map())}>
<span class="icon-x" aria-hidden="true" />
on:click={clear}>
<span class="icon-x" aria-hidden="true"></span>
</button>
</header>
{#each [...importItems.entries()] as [key, value]}
{#each [...$importItems.entries()] as [key, value] (key)}
<div class="upload-box-content" class:is-open={isOpen}>
<ul class="upload-box-list">
<li class="upload-box-item">
@@ -110,17 +170,20 @@
<div
class="progress-bar-top-line u-flex u-gap-8 u-main-space-between">
<Typography.Text>
{text(key)}
{text(value.status)}
</Typography.Text>
<Typography.Caption variant="400">
Collection name
</Typography.Caption>
{#if value.collection}
<Typography.Caption variant="400">
{value.collection}
</Typography.Caption>
{/if}
</div>
<div
class="progress-bar-container"
class:is-danger={key.status === 'failed'}
style="--graph-size:{graphSize(key.status)}%" />
class:is-danger={value.status === 'failed'}
style="--graph-size:{graphSize(value.status)}%">
</div>
</section>
</li>
</ul>
+2 -2
View File
@@ -205,10 +205,10 @@
disabled={uploading}
on:click={() => fileSelector.click()}>
<input
tabindex="-1"
type="file"
accept="image/*"
tabindex="-1"
class="u-hide"
accept={mimeTypeQuery}
on:change={uploadFile}
bind:this={fileSelector} />
{#if uploading}
@@ -25,7 +25,7 @@
<Label {optionalText} {tooltip} hide={!label}>
{label}{#if $$slots.popover && isPopoverDefined}
<Drop bind:show display="inline-block">
<!-- TODO: make unclicked icon greyed out and hover and clicked filled -->
<!-- TODO: make un-clicked icon greyed out and hover and clicked filled -->
&nbsp;<button
type="button"
on:click={() => (show = !show)}
-18
View File
@@ -97,24 +97,6 @@ export function removeFile(file: File, files: FileList) {
return dataTransfer.files;
}
export function readColumnsFromCSV(file: File): Promise<string[]> {
// more than enough!
const CHUNK_SIZE = 4096;
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const text = reader.result as string;
const [firstLine] = text.split('\n');
resolve(firstLine.split(','));
};
reader.onerror = reject;
reader.readAsText(file.slice(0, CHUNK_SIZE));
});
}
export const defaultIgnore = `
### Node ###
# Logs
@@ -8,32 +8,24 @@
import { Container } from '$lib/layout';
import { preferences } from '$lib/stores/preferences';
import { canWriteCollections, canWriteDocuments } from '$lib/stores/roles';
import {
Card,
Icon,
Layout,
Empty as PinkEmpty,
Typography,
Table as PinkTable,
Tooltip,
Upload
} from '@appwrite.io/pink-svelte';
import { Card, Icon, Layout, Empty as PinkEmpty } from '@appwrite.io/pink-svelte';
import type { PageData } from './$types';
import CreateAttributeDropdown from './attributes/createAttributeDropdown.svelte';
import type { Option } from './attributes/store';
import CreateAttribute from './createAttribute.svelte';
import { collection, columns, isCsvImportInProgress } from './store';
import Table from './table.svelte';
import { IconInfo, IconPlus } from '@appwrite.io/pink-icons-svelte';
import { IconPlus } from '@appwrite.io/pink-icons-svelte';
import { base } from '$app/paths';
import { Submit, trackEvent } from '$lib/actions/analytics';
import { readColumnsFromCSV, removeFile } from '$lib/helpers/files';
import { Click, Submit, trackError, trackEvent } from '$lib/actions/analytics';
import FilePicker from '$lib/components/filePicker.svelte';
import type { Models } from '@appwrite.io/console';
import { sdk } from '$lib/stores/sdk';
import { addNotification } from '$lib/stores/notifications';
export let data: PageData;
let files: FileList;
let showImportCSV = false;
let showValidityAlert = false;
let showCreateAttribute = false;
let selectedAttribute: Option['name'] = null;
@@ -52,37 +44,31 @@
$: hasAttributes = !!$collection.attributes.length;
$: hasValidAttributes = $collection?.attributes?.some((attr) => attr.status === 'available');
let differences: Record<string, string[]> = {};
async function checkCsvAttrsValidity() {
if (!files.length) {
differences = { csv: [], collection: [] };
return;
async function onSelect(file: Models.File) {
$isCsvImportInProgress = true;
try {
await sdk.forProject.migrations.createCsvMigration(
file.bucketId,
file.$id,
`${page.params.database}:${page.params.collection}`
);
addNotification({
type: 'success',
message: 'Documents import from csv has started'
});
trackEvent(Submit.DatabaseImportCsv);
} catch (e) {
trackError(e, Submit.DatabaseImportCsv);
addNotification({
type: 'error',
message: e.message
});
} finally {
$isCsvImportInProgress = false;
}
const columns = await readColumnsFromCSV(files[0]);
const expected = $collection.attributes.map((attr) => attr.key);
showValidityAlert = !(
columns.length === expected.length && columns.every((col, i) => col === expected[i])
);
if (showValidityAlert) {
differences = {
csv: Array.from({ length: columns.length }, (_, i) => columns[i] || ''),
collection: Array.from(
{ length: $collection.attributes.length },
(_, i) => expected[i] || ''
)
};
} else {
differences = { csv: [], collection: [] };
}
}
$: if (!showImportCSV) {
files = null;
showValidityAlert = false;
differences = { csv: [], collection: [] };
}
</script>
@@ -98,7 +84,7 @@
<ViewSelector view={data.view} {columns} hideView />
<Button
secondary
event="import_documents"
event={Click.DatabaseImportCsv}
disabled={!(hasAttributes && hasValidAttributes)}
on:click={() => (showImportCSV = true)}>
Import CSV
@@ -203,75 +189,7 @@
bind:selectedOption={selectedAttribute} />
{/if}
<Modal
title="Import CSV"
bind:show={showImportCSV}
onSubmit={() => {
// todo: trigger upload and import once SDK is available.
showImportCSV = false;
$isCsvImportInProgress = true;
}}>
<Typography.Text slot="description">
Select a CSV file to upload. Ensure the file is properly formatted before proceeding.
<!-- TODO: @itznotabug, need a docs link here -->
<Button compact href="https://appwrite.io/docs/products/databases">Learn more.</Button>
</Typography.Text>
<Upload.Dropzone
required
bind:files
title="Upload CSV"
extensions={['csv']}
on:change={checkCsvAttrsValidity}>
<Layout.Stack alignItems="center" gap="s">
<Layout.Stack alignItems="center" gap="s">
<Layout.Stack alignItems="center" justifyContent="center" direction="row" gap="s">
<Typography.Text variant="l-500">
Drag and drop file here or click to upload
</Typography.Text>
<Tooltip maxWidth="fit-content" placement="top">
<Layout.Stack alignItems="center" justifyContent="center" inline>
<Icon icon={IconInfo} size="s" />
</Layout.Stack>
<svelte:fragment slot="tooltip">Only csv files allowed</svelte:fragment>
</Tooltip>
</Layout.Stack>
</Layout.Stack>
</Layout.Stack>
</Upload.Dropzone>
{#if files?.length}
<!-- TODO: torsten, the types issue with FileList-->
<Upload.List bind:files on:remove={(e) => (files = removeFile(e.detail, files))} />
{/if}
{#if showValidityAlert}
<!-- TODO: @itznotabug, need to brainstorm for design on this one -->
<Layout.Stack gap="s">
<Typography.Text>
The uploaded CSV does not match the collection's attribute structure.
</Typography.Text>
<PinkTable.Root columns={2} let:root stickyHeaders maxHeight="16rem">
<svelte:fragment slot="header" let:root>
<PinkTable.Header.Cell column="csv" {root}>CSV</PinkTable.Header.Cell>
<PinkTable.Header.Cell column="collection" {root}
>Collection</PinkTable.Header.Cell>
</svelte:fragment>
{#each differences.csv as csvKey, i}
<PinkTable.Row.Base {root}>
<PinkTable.Cell column="csv" {root}>{csvKey ?? '-'}</PinkTable.Cell>
<PinkTable.Cell column="collection" {root}
>{differences.collection[i] ?? '-'}</PinkTable.Cell>
</PinkTable.Row.Base>
{/each}
</PinkTable.Root>
</Layout.Stack>
{/if}
<svelte:fragment slot="footer">
<Button secondary on:click={() => (showImportCSV = false)}>Cancel</Button>
<Button
disabled={files ? (showValidityAlert ? showValidityAlert : files?.length === 0) : true}
submit>Import</Button>
</svelte:fragment>
</Modal>
{#if showImportCSV}
<!-- CSVs can be text/plain or text/csv sometimes! -->
<FilePicker {onSelect} mimeTypeQuery="text/" bind:show={showImportCSV} />
{/if}
@@ -24,7 +24,7 @@
);
const selectedCollection = $derived.by(() =>
sortedCollections()?.find((collection) => collection.$id === collectionId)
sortedCollections?.find((collection) => collection.$id === collectionId)
);
let openBottomSheet = $state(false);
@@ -3,7 +3,7 @@
import { base } from '$app/paths';
import { page } from '$app/state';
import { Submit, trackError, trackEvent } from '$lib/actions/analytics';
import { Alert, Id, Modal } from '$lib/components';
import { Alert, Confirm, Id } from '$lib/components';
import { Dependencies } from '$lib/constants';
import { Button as ConsoleButton, InputChoice } from '$lib/elements/forms';
import { addNotification } from '$lib/stores/notifications';
@@ -112,7 +112,6 @@
let selectedRows: string[] = [];
let showDelete = false;
let deleting = false;
async function handleDelete() {
showDelete = false;
@@ -267,11 +266,7 @@
</FloatingActionBar>
{/if}
<Modal
title="Delete Documents"
bind:show={showDelete}
onSubmit={handleDelete}
dismissible={!deleting}>
<Confirm title="Delete Documents" bind:open={showDelete} onSubmit={handleDelete}>
<div>
Are you sure you want to delete <b>{selectedRows.length}</b>
{selectedRows.length > 1 ? 'documents' : 'document'}?
@@ -323,12 +318,4 @@
<p class="u-bold">This action is irreversible.</p>
{/if}
</div>
<svelte:fragment slot="footer">
<ConsoleButton text on:click={() => (showDelete = false)} disabled={deleting}
>Cancel</ConsoleButton>
<ConsoleButton secondary submit disabled={deleting || (relAttributes?.length && !checked)}>
Delete
</ConsoleButton>
</svelte:fragment>
</Modal>
</Confirm>
@@ -89,7 +89,7 @@
</script>
<Form onSubmit={update}>
<CardGrid hideFooter={message.status != 'draft'}>
<CardGrid hideFooter={message.status !== 'draft'}>
<div class="grid-1-2-col-1 u-flex-vertical u-cross-start u-gap-16">
<Typography.Title size="s">Message</Typography.Title>
<div class="u-flex u-margin-block-start-24 u-width-full-line">