mirror of
https://github.com/appwrite/console.git
synced 2026-11-18 20:47:21 +00:00
update: realtime progress on csv imports!
This commit is contained in:
1 parent
a35a307d78
commit
014d07951d
11 files changed
+833
-1101
No files matched your search
+1
-1
@@ -21,7 +21,7 @@
|
||||
"e2e:ui": "playwright test --ui"
|
||||
},
|
||||
"dependencies": {
|
||||
"@appwrite.io/console": "https://pkg.pr.new/appwrite/appwrite/@appwrite.io/console@e2f082e",
|
||||
"@appwrite.io/console": "https://pkg.pr.new/appwrite/appwrite/@appwrite.io/console@61868a9",
|
||||
"@appwrite.io/pink-icons": "0.25.0",
|
||||
"@appwrite.io/pink-icons-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@6b17cb23",
|
||||
"@appwrite.io/pink-legacy": "^1.0.3",
|
||||
|
||||
Generated
+678
-898
File diff suppressed because it is too large.
Load diff
@@ -149,6 +149,7 @@ export enum Click {
|
||||
DatabaseIndexDelete = 'click_index_delete',
|
||||
DatabaseCollectionDelete = 'click_collection_delete',
|
||||
DatabaseDatabaseDelete = 'click_database_delete',
|
||||
DatabaseImportCsv = 'click_database_import_csv',
|
||||
DomainCreateClick = 'click_domain_create',
|
||||
DomainDeleteClick = 'click_domain_delete',
|
||||
DomainRetryDomainVerificationClick = 'click_domain_retry_domain_verification',
|
||||
@@ -267,6 +268,7 @@ export enum Submit {
|
||||
DatabaseCreate = 'submit_database_create',
|
||||
DatabaseDelete = 'submit_database_delete',
|
||||
DatabaseUpdateName = 'submit_database_update_name',
|
||||
DatabaseImportCsv = 'submit_database_import_csv',
|
||||
AttributeCreate = 'submit_attribute_create',
|
||||
AttributeUpdate = 'submit_attribute_update',
|
||||
AttributeDelete = 'submit_attribute_delete',
|
||||
|
||||
@@ -1,34 +1,95 @@
|
||||
<script lang="ts">
|
||||
import { type Payload } from '@appwrite.io/console';
|
||||
import { onMount } from 'svelte';
|
||||
import { goto } from '$app/navigation';
|
||||
import { addNotification } from '$lib/stores/notifications';
|
||||
import { base } from '$app/paths';
|
||||
import { page } from '$app/state';
|
||||
import { sdk } from '$lib/stores/sdk';
|
||||
import { Dependencies } from '$lib/constants';
|
||||
import { goto, invalidate } from '$app/navigation';
|
||||
import { getProjectId } from '$lib/helpers/project';
|
||||
import { Typography } from '@appwrite.io/pink-svelte';
|
||||
import { page } from '$app/stores';
|
||||
let importItems: Map<string, string> = new Map<string, string>();
|
||||
import { writable, type Writable } from 'svelte/store';
|
||||
import { addNotification } from '$lib/stores/notifications';
|
||||
import { type Models, type Payload, Query } from '@appwrite.io/console';
|
||||
|
||||
type ImportItem = {
|
||||
status: string;
|
||||
collection?: string;
|
||||
};
|
||||
|
||||
type ImportItemsMap = Map<string, ImportItem>;
|
||||
|
||||
/**
|
||||
* Keeps a track of the active and ongoing csv migrations.
|
||||
*
|
||||
* The structure is as follows -
|
||||
* `{ migrationId: { status: status, collection: collection } }`
|
||||
*/
|
||||
const importItems: Writable<ImportItemsMap> = writable(new Map());
|
||||
|
||||
async function showCompletionNotification(databaseId: string, collectionId: string) {
|
||||
const projectId = page.params.project;
|
||||
await invalidate(Dependencies.DOCUMENTS);
|
||||
const url = `${base}/project-${projectId}/databases/database-${databaseId}/collection-${collectionId}`;
|
||||
|
||||
function showCompletionNotification(databaseId: string, collectionId: string) {
|
||||
const projectId = $page.params.project;
|
||||
addNotification({
|
||||
type: 'success',
|
||||
isHtml: true,
|
||||
message: `Documents import through CSV complete.`,
|
||||
buttons: [
|
||||
{
|
||||
name: 'View imported documents',
|
||||
method: () => {
|
||||
goto(
|
||||
`${base}/project-${projectId}/databases/database-${databaseId}/collection-${collectionId}`
|
||||
);
|
||||
}
|
||||
}
|
||||
]
|
||||
message: `CSV import finished successfully.`,
|
||||
buttons:
|
||||
collectionId === page.params.collection
|
||||
? undefined
|
||||
: [
|
||||
{
|
||||
name: 'View documents',
|
||||
method: () => goto(url)
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
// TODO: @itznotabug, update after the SDK is available
|
||||
function updateOrAddItem(payload: Payload) {}
|
||||
async function updateOrAddItem(importData: Payload | Models.Migration) {
|
||||
if (importData.source.toLowerCase() !== 'csv') return;
|
||||
|
||||
const existing = $importItems.get(importData.$id);
|
||||
if (existing?.status === importData.status) return;
|
||||
|
||||
const status = importData.status;
|
||||
const resourceId = importData.resourceId ?? '';
|
||||
const [databaseId, collectionId] = resourceId.split(':') ?? [];
|
||||
|
||||
let collectionName = existing?.collection ?? null;
|
||||
|
||||
console.log([importData.status, text(importData.status)].join(', '));
|
||||
|
||||
// fetch if we don't have it
|
||||
if (!existing && collectionId) {
|
||||
try {
|
||||
const collection = await sdk.forProject.databases.getCollection(
|
||||
databaseId,
|
||||
collectionId
|
||||
);
|
||||
collectionName = collection.name;
|
||||
} catch (err) {
|
||||
collectionName = null;
|
||||
}
|
||||
}
|
||||
|
||||
importItems.update((items) => {
|
||||
items.set(importData.$id, { status, collection: collectionName });
|
||||
return items;
|
||||
});
|
||||
|
||||
if (status === 'completed') {
|
||||
await showCompletionNotification(databaseId, collectionId);
|
||||
}
|
||||
}
|
||||
|
||||
function clear() {
|
||||
importItems.update((items) => {
|
||||
items.clear();
|
||||
return items;
|
||||
});
|
||||
}
|
||||
|
||||
function graphSize(status: string): number {
|
||||
switch (status) {
|
||||
@@ -57,23 +118,22 @@
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
// Mocking data
|
||||
//showCsvImportBox = true;
|
||||
// importItems.set(ID.unique(), 'something here')
|
||||
// importItems.set(ID.unique(), 'something here 2')
|
||||
// no idea yet on what events to observe
|
||||
// return sdk.forConsole.client.subscribe('console', (response) => {
|
||||
// if (!response.channels.includes(`projects.${getProjectId()}`)) return;
|
||||
//
|
||||
// // TODO: @itznotabug, update after SDK is available
|
||||
// if (response.events.includes('*')) {
|
||||
// updateOrAddItem(response.payload);
|
||||
// }
|
||||
// });
|
||||
sdk.forProject.migrations
|
||||
.list([Query.equal('source', 'CSV'), Query.equal('status', ['pending', 'processing'])])
|
||||
.then((migrations) => {
|
||||
migrations.migrations.forEach(updateOrAddItem);
|
||||
});
|
||||
|
||||
return sdk.forConsole.client.subscribe('console', (response) => {
|
||||
if (!response.channels.includes(`projects.${getProjectId()}`)) return;
|
||||
if (response.events.includes('migrations.*')) {
|
||||
updateOrAddItem(response.payload);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
$: isOpen = true;
|
||||
$: showCsvImportBox = importItems.size > 0;
|
||||
$: showCsvImportBox = $importItems.size > 0;
|
||||
</script>
|
||||
|
||||
{#if showCsvImportBox}
|
||||
@@ -82,7 +142,7 @@
|
||||
<header class="upload-box-header">
|
||||
<h4 class="upload-box-title">
|
||||
<Typography.Text variant="m-500">
|
||||
Importing documents ({importItems.size})
|
||||
Importing documents ({$importItems.size})
|
||||
</Typography.Text>
|
||||
</h4>
|
||||
<button
|
||||
@@ -92,17 +152,17 @@
|
||||
on:click={() => {
|
||||
isOpen = !isOpen;
|
||||
}}>
|
||||
<span class="icon-cheveron-up" aria-hidden="true" />
|
||||
<span class="icon-cheveron-up" aria-hidden="true"></span>
|
||||
</button>
|
||||
<button
|
||||
class="upload-box-button"
|
||||
aria-label="close backup restore box"
|
||||
on:click={() => (importItems = new Map())}>
|
||||
<span class="icon-x" aria-hidden="true" />
|
||||
on:click={clear}>
|
||||
<span class="icon-x" aria-hidden="true"></span>
|
||||
</button>
|
||||
</header>
|
||||
|
||||
{#each [...importItems.entries()] as [key, value]}
|
||||
{#each [...$importItems.entries()] as [key, value] (key)}
|
||||
<div class="upload-box-content" class:is-open={isOpen}>
|
||||
<ul class="upload-box-list">
|
||||
<li class="upload-box-item">
|
||||
@@ -110,17 +170,20 @@
|
||||
<div
|
||||
class="progress-bar-top-line u-flex u-gap-8 u-main-space-between">
|
||||
<Typography.Text>
|
||||
{text(key)}
|
||||
{text(value.status)}
|
||||
</Typography.Text>
|
||||
|
||||
<Typography.Caption variant="400">
|
||||
Collection name
|
||||
</Typography.Caption>
|
||||
{#if value.collection}
|
||||
<Typography.Caption variant="400">
|
||||
{value.collection}
|
||||
</Typography.Caption>
|
||||
{/if}
|
||||
</div>
|
||||
<div
|
||||
class="progress-bar-container"
|
||||
class:is-danger={key.status === 'failed'}
|
||||
style="--graph-size:{graphSize(key.status)}%" />
|
||||
class:is-danger={value.status === 'failed'}
|
||||
style="--graph-size:{graphSize(value.status)}%">
|
||||
</div>
|
||||
</section>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
@@ -205,10 +205,10 @@
|
||||
disabled={uploading}
|
||||
on:click={() => fileSelector.click()}>
|
||||
<input
|
||||
tabindex="-1"
|
||||
type="file"
|
||||
accept="image/*"
|
||||
tabindex="-1"
|
||||
class="u-hide"
|
||||
accept={mimeTypeQuery}
|
||||
on:change={uploadFile}
|
||||
bind:this={fileSelector} />
|
||||
{#if uploading}
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<Label {optionalText} {tooltip} hide={!label}>
|
||||
{label}{#if $$slots.popover && isPopoverDefined}
|
||||
<Drop bind:show display="inline-block">
|
||||
<!-- TODO: make unclicked icon greyed out and hover and clicked filled -->
|
||||
<!-- TODO: make un-clicked icon greyed out and hover and clicked filled -->
|
||||
<button
|
||||
type="button"
|
||||
on:click={() => (show = !show)}
|
||||
|
||||
@@ -97,24 +97,6 @@ export function removeFile(file: File, files: FileList) {
|
||||
return dataTransfer.files;
|
||||
}
|
||||
|
||||
export function readColumnsFromCSV(file: File): Promise<string[]> {
|
||||
// more than enough!
|
||||
const CHUNK_SIZE = 4096;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
|
||||
reader.onload = () => {
|
||||
const text = reader.result as string;
|
||||
const [firstLine] = text.split('\n');
|
||||
resolve(firstLine.split(','));
|
||||
};
|
||||
|
||||
reader.onerror = reject;
|
||||
reader.readAsText(file.slice(0, CHUNK_SIZE));
|
||||
});
|
||||
}
|
||||
|
||||
export const defaultIgnore = `
|
||||
### Node ###
|
||||
# Logs
|
||||
|
||||
+36
-118
@@ -8,32 +8,24 @@
|
||||
import { Container } from '$lib/layout';
|
||||
import { preferences } from '$lib/stores/preferences';
|
||||
import { canWriteCollections, canWriteDocuments } from '$lib/stores/roles';
|
||||
import {
|
||||
Card,
|
||||
Icon,
|
||||
Layout,
|
||||
Empty as PinkEmpty,
|
||||
Typography,
|
||||
Table as PinkTable,
|
||||
Tooltip,
|
||||
Upload
|
||||
} from '@appwrite.io/pink-svelte';
|
||||
import { Card, Icon, Layout, Empty as PinkEmpty } from '@appwrite.io/pink-svelte';
|
||||
import type { PageData } from './$types';
|
||||
import CreateAttributeDropdown from './attributes/createAttributeDropdown.svelte';
|
||||
import type { Option } from './attributes/store';
|
||||
import CreateAttribute from './createAttribute.svelte';
|
||||
import { collection, columns, isCsvImportInProgress } from './store';
|
||||
import Table from './table.svelte';
|
||||
import { IconInfo, IconPlus } from '@appwrite.io/pink-icons-svelte';
|
||||
import { IconPlus } from '@appwrite.io/pink-icons-svelte';
|
||||
import { base } from '$app/paths';
|
||||
import { Submit, trackEvent } from '$lib/actions/analytics';
|
||||
import { readColumnsFromCSV, removeFile } from '$lib/helpers/files';
|
||||
import { Click, Submit, trackError, trackEvent } from '$lib/actions/analytics';
|
||||
import FilePicker from '$lib/components/filePicker.svelte';
|
||||
import type { Models } from '@appwrite.io/console';
|
||||
import { sdk } from '$lib/stores/sdk';
|
||||
import { addNotification } from '$lib/stores/notifications';
|
||||
|
||||
export let data: PageData;
|
||||
|
||||
let files: FileList;
|
||||
let showImportCSV = false;
|
||||
let showValidityAlert = false;
|
||||
let showCreateAttribute = false;
|
||||
let selectedAttribute: Option['name'] = null;
|
||||
|
||||
@@ -52,37 +44,31 @@
|
||||
$: hasAttributes = !!$collection.attributes.length;
|
||||
$: hasValidAttributes = $collection?.attributes?.some((attr) => attr.status === 'available');
|
||||
|
||||
let differences: Record<string, string[]> = {};
|
||||
async function checkCsvAttrsValidity() {
|
||||
if (!files.length) {
|
||||
differences = { csv: [], collection: [] };
|
||||
return;
|
||||
async function onSelect(file: Models.File) {
|
||||
$isCsvImportInProgress = true;
|
||||
|
||||
try {
|
||||
await sdk.forProject.migrations.createCsvMigration(
|
||||
file.bucketId,
|
||||
file.$id,
|
||||
`${page.params.database}:${page.params.collection}`
|
||||
);
|
||||
|
||||
addNotification({
|
||||
type: 'success',
|
||||
message: 'Documents import from csv has started'
|
||||
});
|
||||
|
||||
trackEvent(Submit.DatabaseImportCsv);
|
||||
} catch (e) {
|
||||
trackError(e, Submit.DatabaseImportCsv);
|
||||
addNotification({
|
||||
type: 'error',
|
||||
message: e.message
|
||||
});
|
||||
} finally {
|
||||
$isCsvImportInProgress = false;
|
||||
}
|
||||
|
||||
const columns = await readColumnsFromCSV(files[0]);
|
||||
const expected = $collection.attributes.map((attr) => attr.key);
|
||||
|
||||
showValidityAlert = !(
|
||||
columns.length === expected.length && columns.every((col, i) => col === expected[i])
|
||||
);
|
||||
|
||||
if (showValidityAlert) {
|
||||
differences = {
|
||||
csv: Array.from({ length: columns.length }, (_, i) => columns[i] || ''),
|
||||
collection: Array.from(
|
||||
{ length: $collection.attributes.length },
|
||||
(_, i) => expected[i] || ''
|
||||
)
|
||||
};
|
||||
} else {
|
||||
differences = { csv: [], collection: [] };
|
||||
}
|
||||
}
|
||||
|
||||
$: if (!showImportCSV) {
|
||||
files = null;
|
||||
showValidityAlert = false;
|
||||
differences = { csv: [], collection: [] };
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -98,7 +84,7 @@
|
||||
<ViewSelector view={data.view} {columns} hideView />
|
||||
<Button
|
||||
secondary
|
||||
event="import_documents"
|
||||
event={Click.DatabaseImportCsv}
|
||||
disabled={!(hasAttributes && hasValidAttributes)}
|
||||
on:click={() => (showImportCSV = true)}>
|
||||
Import CSV
|
||||
@@ -203,75 +189,7 @@
|
||||
bind:selectedOption={selectedAttribute} />
|
||||
{/if}
|
||||
|
||||
<Modal
|
||||
title="Import CSV"
|
||||
bind:show={showImportCSV}
|
||||
onSubmit={() => {
|
||||
// todo: trigger upload and import once SDK is available.
|
||||
showImportCSV = false;
|
||||
$isCsvImportInProgress = true;
|
||||
}}>
|
||||
<Typography.Text slot="description">
|
||||
Select a CSV file to upload. Ensure the file is properly formatted before proceeding.
|
||||
<!-- TODO: @itznotabug, need a docs link here -->
|
||||
<Button compact href="https://appwrite.io/docs/products/databases">Learn more.</Button>
|
||||
</Typography.Text>
|
||||
|
||||
<Upload.Dropzone
|
||||
required
|
||||
bind:files
|
||||
title="Upload CSV"
|
||||
extensions={['csv']}
|
||||
on:change={checkCsvAttrsValidity}>
|
||||
<Layout.Stack alignItems="center" gap="s">
|
||||
<Layout.Stack alignItems="center" gap="s">
|
||||
<Layout.Stack alignItems="center" justifyContent="center" direction="row" gap="s">
|
||||
<Typography.Text variant="l-500">
|
||||
Drag and drop file here or click to upload
|
||||
</Typography.Text>
|
||||
<Tooltip maxWidth="fit-content" placement="top">
|
||||
<Layout.Stack alignItems="center" justifyContent="center" inline>
|
||||
<Icon icon={IconInfo} size="s" />
|
||||
</Layout.Stack>
|
||||
<svelte:fragment slot="tooltip">Only csv files allowed</svelte:fragment>
|
||||
</Tooltip>
|
||||
</Layout.Stack>
|
||||
</Layout.Stack>
|
||||
</Layout.Stack>
|
||||
</Upload.Dropzone>
|
||||
{#if files?.length}
|
||||
<!-- TODO: torsten, the types issue with FileList-->
|
||||
<Upload.List bind:files on:remove={(e) => (files = removeFile(e.detail, files))} />
|
||||
{/if}
|
||||
|
||||
{#if showValidityAlert}
|
||||
<!-- TODO: @itznotabug, need to brainstorm for design on this one -->
|
||||
<Layout.Stack gap="s">
|
||||
<Typography.Text>
|
||||
The uploaded CSV does not match the collection's attribute structure.
|
||||
</Typography.Text>
|
||||
|
||||
<PinkTable.Root columns={2} let:root stickyHeaders maxHeight="16rem">
|
||||
<svelte:fragment slot="header" let:root>
|
||||
<PinkTable.Header.Cell column="csv" {root}>CSV</PinkTable.Header.Cell>
|
||||
<PinkTable.Header.Cell column="collection" {root}
|
||||
>Collection</PinkTable.Header.Cell>
|
||||
</svelte:fragment>
|
||||
{#each differences.csv as csvKey, i}
|
||||
<PinkTable.Row.Base {root}>
|
||||
<PinkTable.Cell column="csv" {root}>{csvKey ?? '-'}</PinkTable.Cell>
|
||||
<PinkTable.Cell column="collection" {root}
|
||||
>{differences.collection[i] ?? '-'}</PinkTable.Cell>
|
||||
</PinkTable.Row.Base>
|
||||
{/each}
|
||||
</PinkTable.Root>
|
||||
</Layout.Stack>
|
||||
{/if}
|
||||
|
||||
<svelte:fragment slot="footer">
|
||||
<Button secondary on:click={() => (showImportCSV = false)}>Cancel</Button>
|
||||
<Button
|
||||
disabled={files ? (showValidityAlert ? showValidityAlert : files?.length === 0) : true}
|
||||
submit>Import</Button>
|
||||
</svelte:fragment>
|
||||
</Modal>
|
||||
{#if showImportCSV}
|
||||
<!-- CSVs can be text/plain or text/csv sometimes! -->
|
||||
<FilePicker {onSelect} mimeTypeQuery="text/" bind:show={showImportCSV} />
|
||||
{/if}
|
||||
+1
-1
@@ -24,7 +24,7 @@
|
||||
);
|
||||
|
||||
const selectedCollection = $derived.by(() =>
|
||||
sortedCollections()?.find((collection) => collection.$id === collectionId)
|
||||
sortedCollections?.find((collection) => collection.$id === collectionId)
|
||||
);
|
||||
|
||||
let openBottomSheet = $state(false);
|
||||
|
||||
+3
-16
@@ -3,7 +3,7 @@
|
||||
import { base } from '$app/paths';
|
||||
import { page } from '$app/state';
|
||||
import { Submit, trackError, trackEvent } from '$lib/actions/analytics';
|
||||
import { Alert, Id, Modal } from '$lib/components';
|
||||
import { Alert, Confirm, Id } from '$lib/components';
|
||||
import { Dependencies } from '$lib/constants';
|
||||
import { Button as ConsoleButton, InputChoice } from '$lib/elements/forms';
|
||||
import { addNotification } from '$lib/stores/notifications';
|
||||
@@ -112,7 +112,6 @@
|
||||
|
||||
let selectedRows: string[] = [];
|
||||
let showDelete = false;
|
||||
let deleting = false;
|
||||
|
||||
async function handleDelete() {
|
||||
showDelete = false;
|
||||
@@ -267,11 +266,7 @@
|
||||
</FloatingActionBar>
|
||||
{/if}
|
||||
|
||||
<Modal
|
||||
title="Delete Documents"
|
||||
bind:show={showDelete}
|
||||
onSubmit={handleDelete}
|
||||
dismissible={!deleting}>
|
||||
<Confirm title="Delete Documents" bind:open={showDelete} onSubmit={handleDelete}>
|
||||
<div>
|
||||
Are you sure you want to delete <b>{selectedRows.length}</b>
|
||||
{selectedRows.length > 1 ? 'documents' : 'document'}?
|
||||
@@ -323,12 +318,4 @@
|
||||
<p class="u-bold">This action is irreversible.</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<svelte:fragment slot="footer">
|
||||
<ConsoleButton text on:click={() => (showDelete = false)} disabled={deleting}
|
||||
>Cancel</ConsoleButton>
|
||||
<ConsoleButton secondary submit disabled={deleting || (relAttributes?.length && !checked)}>
|
||||
Delete
|
||||
</ConsoleButton>
|
||||
</svelte:fragment>
|
||||
</Modal>
|
||||
</Confirm>
|
||||
+1
-1
@@ -89,7 +89,7 @@
|
||||
</script>
|
||||
|
||||
<Form onSubmit={update}>
|
||||
<CardGrid hideFooter={message.status != 'draft'}>
|
||||
<CardGrid hideFooter={message.status !== 'draft'}>
|
||||
<div class="grid-1-2-col-1 u-flex-vertical u-cross-start u-gap-16">
|
||||
<Typography.Title size="s">Message</Typography.Title>
|
||||
<div class="u-flex u-margin-block-start-24 u-width-full-line">
|
||||
|
||||
Reference in new issue
Block a user