feat: Import documents from a CSV.

This commit is contained in:
Darshan
2025-03-27 13:00:09 +05:30
parent 68b2af42d3
commit 8bd1869768
10 changed files with 406 additions and 70 deletions
+2 -2
View File
@@ -21,9 +21,9 @@
"dependencies": {
"@appwrite.io/console": "https://pkg.pr.new/appwrite/appwrite/@appwrite.io/console@8db247c",
"@appwrite.io/pink-icons": "0.25.0",
"@appwrite.io/pink-icons-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@4a5aab5",
"@appwrite.io/pink-icons-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@310",
"@appwrite.io/pink-legacy": "^1.0.3",
"@appwrite.io/pink-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@203ff84",
"@appwrite.io/pink-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@310",
"@popperjs/core": "^2.11.8",
"@sentry/sveltekit": "^8.38.0",
"@stripe/stripe-js": "^3.5.0",
+14 -14
View File
@@ -15,14 +15,14 @@ importers:
specifier: 0.25.0
version: 0.25.0
'@appwrite.io/pink-icons-svelte':
specifier: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@4a5aab5
version: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@4a5aab5(svelte@4.2.19)
specifier: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@310
version: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@310(svelte@4.2.19)
'@appwrite.io/pink-legacy':
specifier: ^1.0.3
version: 1.0.3
'@appwrite.io/pink-svelte':
specifier: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@203ff84
version: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@203ff84(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19)
specifier: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@310
version: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@310(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19)
'@popperjs/core':
specifier: ^2.11.8
version: 2.11.8
@@ -215,14 +215,14 @@ packages:
resolution: {tarball: https://pkg.pr.new/appwrite/appwrite/@appwrite.io/console@8db247c}
version: 1.2.1
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@203ff849e6d6a66529bf690af2e97cade99ef488':
resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@203ff849e6d6a66529bf690af2e97cade99ef488}
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@2db2d48b7ac3f0f2a0a8b90c9d81310183586c2a':
resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@2db2d48b7ac3f0f2a0a8b90c9d81310183586c2a}
version: 1.0.0-next.7
peerDependencies:
svelte: ^4.0.0
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@4a5aab5':
resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@4a5aab5}
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@310':
resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@310}
version: 1.0.0-next.7
peerDependencies:
svelte: ^4.0.0
@@ -236,8 +236,8 @@ packages:
'@appwrite.io/pink-legacy@1.0.3':
resolution: {integrity: sha512-GGde5fmPhs+s6/3aFeMPc/kKADG/gTFkYQSy6oBN8pK0y0XNCLrZZgBv+EBbdhwdtqVEWXa0X85Mv9w7jcIlwQ==}
'@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@203ff84':
resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@203ff84}
'@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@310':
resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@310}
version: 1.0.0-next.85
peerDependencies:
react-dom: ^18.0.0
@@ -4243,11 +4243,11 @@ snapshots:
'@appwrite.io/console@https://pkg.pr.new/appwrite/appwrite/@appwrite.io/console@8db247c': {}
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@203ff849e6d6a66529bf690af2e97cade99ef488(svelte@4.2.19)':
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@2db2d48b7ac3f0f2a0a8b90c9d81310183586c2a(svelte@4.2.19)':
dependencies:
svelte: 4.2.19
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@4a5aab5(svelte@4.2.19)':
'@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@310(svelte@4.2.19)':
dependencies:
svelte: 4.2.19
@@ -4260,9 +4260,9 @@ snapshots:
'@appwrite.io/pink-icons': 1.0.0
the-new-css-reset: 1.11.3
'@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@203ff84(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19)':
'@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@310(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19)':
dependencies:
'@appwrite.io/pink-icons-svelte': https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@203ff849e6d6a66529bf690af2e97cade99ef488(svelte@4.2.19)
'@appwrite.io/pink-icons-svelte': https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@2db2d48b7ac3f0f2a0a8b90c9d81310183586c2a(svelte@4.2.19)
'@floating-ui/dom': 1.6.13
'@melt-ui/pp': 0.3.2(@melt-ui/svelte@0.86.3(svelte@4.2.19))(svelte@4.2.19)
'@melt-ui/svelte': 0.86.3(svelte@4.2.19)
+162
View File
@@ -0,0 +1,162 @@
<script lang="ts">
import { type Payload } from '@appwrite.io/console';
import { onMount } from 'svelte';
import { goto } from '$app/navigation';
import { addNotification } from '$lib/stores/notifications';
import { base } from '$app/paths';
import { Typography } from '@appwrite.io/pink-svelte';
import { page } from '$app/stores';
let importItems: Map<string, string> = new Map<string, string>();
function showCompletionNotification(databaseId: string, collectionId: string) {
const projectId = $page.params.project;
addNotification({
type: 'success',
isHtml: true,
message: `Documents import through CSV complete.`,
buttons: [
{
name: 'View imported documents',
method: () => {
goto(
`${base}/project-${projectId}/databases/database-${databaseId}/collection-${collectionId}`
);
}
}
]
});
}
// TODO: @itznotabug, update after the SDK is available
function updateOrAddItem(payload: Payload) {}
function graphSize(status: string): number {
switch (status) {
case 'pending':
return 10;
case 'processing':
return 30;
case 'uploading':
return 60;
case 'completed':
case 'failed':
return 100;
default:
return 30;
}
}
function text(status: string) {
if (status === 'completed') {
return `CSV import complete`;
} else if (status === 'failed') {
return `CSV import failed`;
} else {
return 'Preparing CSV for import...';
}
}
onMount(() => {
// Mocking data
//showCsvImportBox = true;
// importItems.set(ID.unique(), 'something here')
// importItems.set(ID.unique(), 'something here 2')
// no idea yet on what events to observe
// return sdk.forConsole.client.subscribe('console', (response) => {
// if (!response.channels.includes(`projects.${getProjectId()}`)) return;
//
// // TODO: @itznotabug, update after SDK is available
// if (response.events.includes('*')) {
// updateOrAddItem(response.payload);
// }
// });
});
$: isOpen = true;
$: showCsvImportBox = importItems.size > 0;
</script>
{#if showCsvImportBox}
<div class="box-holder u-flex u-flex-vertical u-gap-16" style="align-items: end">
<section class="upload-box">
<header class="upload-box-header">
<h4 class="upload-box-title">
<Typography.Text variant="m-500">
Importing documents ({importItems.size})
</Typography.Text>
</h4>
<button
class="upload-box-button"
class:is-open={isOpen}
aria-label="toggle upload box"
on:click={() => {
isOpen = !isOpen;
}}>
<span class="icon-cheveron-up" aria-hidden="true" />
</button>
<button
class="upload-box-button"
aria-label="close backup restore box"
on:click={() => (importItems = new Map())}>
<span class="icon-x" aria-hidden="true" />
</button>
</header>
{#each [...importItems.entries()] as [key, value]}
<div class="upload-box-content" class:is-open={isOpen}>
<ul class="upload-box-list">
<li class="upload-box-item">
<section class="progress-bar u-width-full-line">
<div
class="progress-bar-top-line u-flex u-gap-8 u-main-space-between">
<Typography.Text>
{text(key)}
</Typography.Text>
<Typography.Caption variant="400">
Collection name
</Typography.Caption>
</div>
<div
class="progress-bar-container"
class:is-danger={key.status === 'failed'}
style="--graph-size:{graphSize(key.status)}%" />
</section>
</li>
</ul>
</div>
{/each}
</section>
</div>
{/if}
<style lang="scss">
.upload-box-title {
font-size: 11px;
}
.upload-box-content {
min-width: 400px;
max-width: 100vw;
}
.upload-box-button {
display: flex;
align-items: center;
justify-content: center;
}
.progress-bar-container {
height: 4px;
&::before {
height: 4px;
background-color: var(--bgcolor-neutral-invert);
}
&.is-danger::before {
height: 4px;
background-color: var(--bgcolor-error);
}
}
</style>
+18
View File
@@ -97,6 +97,24 @@ export function removeFile(file: File, files: FileList) {
return dataTransfer.files;
}
export function readColumnsFromCSV(file: File): Promise<string[]> {
// more than enough!
const CHUNK_SIZE = 4096;
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const text = reader.result as string;
const [firstLine] = text.split('\n');
resolve(firstLine.split(','));
};
reader.onerror = reject;
reader.readAsText(file.slice(0, CHUNK_SIZE));
});
}
export const defaultIgnore = `
### Node ###
# Logs
@@ -24,6 +24,7 @@
canWriteProjects,
canWriteSites
} from '$lib/stores/roles';
import CsvImportBox from '$lib/components/csvImportBox.svelte';
onMount(() => {
return sdk.forConsole.client.subscribe(['project', 'console'], (response) => {
@@ -117,6 +118,7 @@
<UploadBox />
<MigrationBox />
<BackupRestoreBox />
<CsvImportBox />
</div>
<style>
@@ -1,6 +1,6 @@
<script lang="ts">
import { page } from '$app/stores';
import { Empty, EmptySearch, PaginationWithLimit } from '$lib/components';
import { Empty, EmptySearch, Modal, PaginationWithLimit } from '$lib/components';
import { Filters, hasPageQueries, queries } from '$lib/components/filters';
import ViewSelector from '$lib/components/viewSelector.svelte';
import { Button } from '$lib/elements/forms';
@@ -8,19 +8,32 @@
import { Container } from '$lib/layout';
import { preferences } from '$lib/stores/preferences';
import { canWriteCollections, canWriteDocuments } from '$lib/stores/roles';
import { Card, Icon, Layout, Empty as PinkEmpty } from '@appwrite.io/pink-svelte';
import {
Card,
Icon,
Layout,
Empty as PinkEmpty,
Typography,
Table as PinkTable,
Tooltip,
Upload
} from '@appwrite.io/pink-svelte';
import type { PageData } from './$types';
import CreateAttributeDropdown from './attributes/createAttributeDropdown.svelte';
import type { Option } from './attributes/store';
import CreateAttribute from './createAttribute.svelte';
import { collection, columns } from './store';
import Table from './table.svelte';
import { IconPlus } from '@appwrite.io/pink-icons-svelte';
import { IconInfo, IconPlus } from '@appwrite.io/pink-icons-svelte';
import { base } from '$app/paths';
import { Submit, trackEvent } from '$lib/actions/analytics';
import { readColumnsFromCSV, removeFile } from '$lib/helpers/files';
export let data: PageData;
let files: FileList;
let showImportCSV = false;
let showValidityAlert = false;
let showCreateAttribute = false;
let selectedAttribute: Option['name'] = null;
@@ -38,6 +51,39 @@
);
$: hasAttributes = !!$collection.attributes.length;
$: hasValidAttributes = $collection?.attributes?.some((attr) => attr.status === 'available');
let differences: Record<string, string[]> = {};
async function checkCsvAttrsValidity() {
if (!files.length) {
differences = { csv: [], collection: [] };
return;
}
const columns = await readColumnsFromCSV(files[0]);
const expected = $collection.attributes.map((attr) => attr.key);
showValidityAlert = !(
columns.length === expected.length && columns.every((col, i) => col === expected[i])
);
if (showValidityAlert) {
differences = {
csv: Array.from({ length: columns.length }, (_, i) => columns[i] || ''),
collection: Array.from(
{ length: $collection.attributes.length },
(_, i) => expected[i] || ''
)
};
} else {
differences = { csv: [], collection: [] };
}
}
$: if (!showImportCSV) {
files = null;
showValidityAlert = false;
differences = { csv: [], collection: [] };
}
</script>
{#key $page.params.collection}
@@ -50,6 +96,13 @@
analyticsSource="database_documents" />
<Layout.Stack direction="row" alignItems="center" justifyContent="flex-end">
<ViewSelector view={data.view} {columns} hideView allowNoColumns />
<Button
secondary
event="import_documents"
disabled={!(hasAttributes && hasValidAttributes)}
on:click={() => (showImportCSV = true)}>
Import CSV
</Button>
<Button
disabled={!(hasAttributes && hasValidAttributes)}
href={`${base}/project-${$page.params.project}/databases/database-${$page.params.database}/collection-${$page.params.collection}/create`}
@@ -149,3 +202,69 @@
bind:showCreate={showCreateAttribute}
bind:selectedOption={selectedAttribute} />
{/if}
<Modal title="Import CSV" bind:show={showImportCSV}>
<Typography.Text slot="description">
Select a CSV file to upload. Ensure the file is properly formatted before proceeding.
<!-- TODO: @itznotabug, need a docs link here -->
<Button compact href="https://appwrite.io/docs/products/databases">Learn more.</Button>
</Typography.Text>
<Upload.Dropzone
required
bind:files
title="Upload CSV"
extensions={['csv']}
on:change={checkCsvAttrsValidity}>
<Layout.Stack alignItems="center" gap="s">
<Layout.Stack alignItems="center" gap="s">
<Layout.Stack alignItems="center" justifyContent="center" direction="row" gap="s">
<Typography.Text variant="l-500">
Drag and drop file here or click to upload
</Typography.Text>
<Tooltip maxWidth="fit-content" placement="top">
<Layout.Stack alignItems="center" justifyContent="center" inline>
<Icon icon={IconInfo} size="s" />
</Layout.Stack>
<svelte:fragment slot="tooltip">Only csv files allowed</svelte:fragment>
</Tooltip>
</Layout.Stack>
</Layout.Stack>
</Layout.Stack>
</Upload.Dropzone>
{#if files?.length}
<!-- TODO: torsten, the types issue with FileList-->
<Upload.List bind:files on:remove={(e) => (files = removeFile(e.detail, files))} />
{/if}
{#if showValidityAlert}
<!-- TODO: @itznotabug, need to brainstorm for design on this one -->
<Layout.Stack gap="s">
<Typography.Text>
The uploaded CSV doesn't match the structure of the collection's attributes.
</Typography.Text>
<PinkTable.Root columns={2} let:root stickyHeaders maxHeight="14rem">
<svelte:fragment slot="header" let:root>
<PinkTable.Header.Cell column="csv" {root}>CSV</PinkTable.Header.Cell>
<PinkTable.Header.Cell column="collection" {root}
>Collection</PinkTable.Header.Cell>
</svelte:fragment>
{#each differences.csv as csvKey, i}
<PinkTable.Row.Base {root}>
<PinkTable.Cell column="csv" {root}>{csvKey ?? '-'}</PinkTable.Cell>
<PinkTable.Cell column="collection" {root}
>{differences.collection[i] ?? '-'}</PinkTable.Cell>
</PinkTable.Row.Base>
{/each}
</PinkTable.Root>
</Layout.Stack>
{/if}
<svelte:fragment slot="footer">
<Button secondary on:click={() => (showImportCSV = false)}>Cancel</Button>
<Button
disabled={files ? (showValidityAlert ? showValidityAlert : files?.length === 0) : true}
submit>Import</Button>
</svelte:fragment>
</Modal>
@@ -19,7 +19,7 @@
import { isRelationship } from '../document-[document]/attributes/store';
import FailedModal from '../failedModal.svelte';
import CreateIndex from '../indexes/createIndex.svelte';
import { attributes, type Attributes } from '../store';
import { attributes, type Attributes, isCsvImportInProgress } from '../store';
import CreateAttributeDropdown from './createAttributeDropdown.svelte';
import Delete from './deleteAttribute.svelte';
import Edit from './edit.svelte';
@@ -36,6 +36,7 @@
} from '@appwrite.io/pink-icons-svelte';
import type { ComponentProps } from 'svelte';
import { Click, trackEvent } from '$lib/actions/analytics';
import CsvDisabled from '../csvDisabled.svelte';
const projectId = $page.params.project;
const databaseId = $page.params.database;
@@ -155,45 +156,53 @@
: '-'}
</Table.Cell>
<Table.Cell column="actions" {root}>
<Popover let:toggle padding="none" placement="bottom-end">
<Button text icon ariaLabel="more options" on:click={toggle}>
<Icon icon={IconDotsHorizontal} size="s" />
</Button>
<ActionMenu.Root slot="tooltip">
<ActionMenu.Item.Button
leadingIcon={IconPencil}
on:click={() => {
selectedAttribute = attribute;
showEdit = true;
showDropdown[index] = false;
}}>
Update
</ActionMenu.Item.Button>
{#if !isRelationship(attribute)}
{#if $isCsvImportInProgress}
<CsvDisabled>
<Button disabled text icon ariaLabel="more options">
<Icon icon={IconDotsHorizontal} size="s" />
</Button>
</CsvDisabled>
{:else}
<Popover let:toggle padding="none" placement="bottom-end">
<Button text icon ariaLabel="more options" on:click={toggle}>
<Icon icon={IconDotsHorizontal} size="s" />
</Button>
<ActionMenu.Root slot="tooltip">
<ActionMenu.Item.Button
leadingIcon={IconPlus}
leadingIcon={IconPencil}
on:click={() => {
selectedAttribute = attribute;
showCreateIndex = true;
showEdit = true;
showDropdown[index] = false;
}}>
Create index
Update
</ActionMenu.Item.Button>
{/if}
{#if attribute.status !== 'processing'}
<ActionMenu.Item.Button
leadingIcon={IconTrash}
on:click={() => {
selectedAttribute = attribute;
showDelete = true;
showDropdown[index] = false;
trackEvent(Click.DatabaseAttributeDelete);
}}>
Delete
</ActionMenu.Item.Button>
{/if}
</ActionMenu.Root>
</Popover>
{#if !isRelationship(attribute)}
<ActionMenu.Item.Button
leadingIcon={IconPlus}
on:click={() => {
selectedAttribute = attribute;
showCreateIndex = true;
showDropdown[index] = false;
}}>
Create index
</ActionMenu.Item.Button>
{/if}
{#if attribute.status !== 'processing'}
<ActionMenu.Item.Button
leadingIcon={IconTrash}
on:click={() => {
selectedAttribute = attribute;
showDelete = true;
showDropdown[index] = false;
trackEvent(Click.DatabaseAttributeDelete);
}}>
Delete
</ActionMenu.Item.Button>
{/if}
</ActionMenu.Root>
</Popover>
{/if}
</Table.Cell>
</Table.Row.Base>
{/each}
@@ -3,28 +3,39 @@
import { ActionMenu, Icon, Popover } from '@appwrite.io/pink-svelte';
import { attributeOptions, type Option } from './store';
import { IconPlus } from '@appwrite.io/pink-icons-svelte';
import CsvDisabled from '../csvDisabled.svelte';
import { isCsvImportInProgress } from '../store';
export let selectedOption: Option['name'] = null;
export let showCreate = false;
</script>
<Popover let:toggle padding="none" placement="bottom-start">
<slot {toggle}>
<Button on:click={toggle} event="create_attribute">
{#if $isCsvImportInProgress}
<CsvDisabled>
<Button size="s" disabled>
<Icon icon={IconPlus} slot="start" size="s" />
Create attribute
</Button>
</slot>
<ActionMenu.Root slot="tooltip">
{#each attributeOptions as attribute}
<ActionMenu.Item.Button
leadingIcon={attribute.icon}
on:click={() => {
selectedOption = attribute.name;
showCreate = true;
}}>
{attribute.name}
</ActionMenu.Item.Button>
{/each}
</ActionMenu.Root>
</Popover>
</CsvDisabled>
{:else}
<Popover let:toggle padding="none" placement="bottom-start">
<slot {toggle}>
<Button on:click={toggle} event="create_attribute">
<Icon icon={IconPlus} slot="start" size="s" />
Create attribute
</Button>
</slot>
<ActionMenu.Root slot="tooltip">
{#each attributeOptions as attribute}
<ActionMenu.Item.Button
leadingIcon={attribute.icon}
on:click={() => {
selectedOption = attribute.name;
showCreate = true;
}}>
{attribute.name}
</ActionMenu.Item.Button>
{/each}
</ActionMenu.Root>
</Popover>
{/if}
@@ -0,0 +1,12 @@
<script lang="ts">
import { isCsvImportInProgress } from './store';
import { Tooltip } from '@appwrite.io/pink-svelte';
</script>
<Tooltip disabled={!$isCsvImportInProgress} maxWidth="12rem">
<div>
<slot />
</div>
<div slot="tooltip">This action is disabled during import.</div>
</Tooltip>
@@ -26,3 +26,6 @@ export const attributes = derived(
export const indexes = derived(page, ($page) => $page.data.collection.indexes as Models.Index[]);
export const columns = writable<Column[]>([]);
// TODO: @itznotabug, check what data is received when SDK is available.
export const isCsvImportInProgress = derived(page, ($page) => $page.data.csvImport ?? true);