From 0c82bd5de92d5bd1eceb5495744bf27c35d9d7d6 Mon Sep 17 00:00:00 2001 From: Arman Date: Thu, 19 Dec 2024 17:13:12 +0100 Subject: [PATCH] feat: manual upload --- package.json | 3 +- pnpm-lock.yaml | 27 ++-- src/lib/components/uploadBox.svelte | 2 +- src/lib/stores/uploader.ts | 55 +++++-- .../createManualDeploymentModal.svelte | 129 +++++++++++----- .../sites/site-[site]/deployments/store.ts | 143 ++++++++++++++++++ 6 files changed, 302 insertions(+), 57 deletions(-) diff --git a/package.json b/package.json index fb9b07157..6b1fe2371 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "@appwrite.io/pink": "0.25.0", "@appwrite.io/pink-icons": "0.25.0", "@appwrite.io/pink-icons-svelte": "1.0.0-next.7", - "@appwrite.io/pink-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@dbeda91", + "@appwrite.io/pink-svelte": "https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@69695cf", "@popperjs/core": "^2.11.8", "@sentry/sveltekit": "^8.38.0", "@stripe/stripe-js": "^3.5.0", @@ -34,6 +34,7 @@ "deep-equal": "^2.2.3", "echarts": "^5.5.1", "envfile": "^7.1.0", + "ignore": "^6.0.2", "nanoid": "^5.0.8", "nanotar": "^0.1.1", "plausible-tracker": "^0.3.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4763f9f51..d8dade7ee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 1.0.0-next.7 version: 1.0.0-next.7(svelte@4.2.19) '@appwrite.io/pink-svelte': - specifier: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@dbeda91 - version: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@dbeda91(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19) + specifier: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@69695cf + version: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@69695cf(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19) '@popperjs/core': specifier: ^2.11.8 version: 2.11.8 @@ -53,6 +53,9 @@ importers: envfile: specifier: ^7.1.0 version: 7.1.0 + ignore: + specifier: ^6.0.2 + version: 6.0.2 nanoid: specifier: ^5.0.8 version: 5.0.9 @@ -214,8 +217,8 @@ packages: peerDependencies: svelte: ^4.0.0 - '@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@dbeda91e16325c2072088627718c0b637263cc76': - resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@dbeda91e16325c2072088627718c0b637263cc76} + '@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@69695cf7bbcf4d49aa40ee2a2cfee6c46f6efac6': + resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@69695cf7bbcf4d49aa40ee2a2cfee6c46f6efac6} version: 1.0.0-next.7 peerDependencies: svelte: ^4.0.0 @@ -223,8 +226,8 @@ packages: '@appwrite.io/pink-icons@0.25.0': resolution: {integrity: sha512-0O3i2oEuh5mWvjO80i+X6rbzrWLJ1m5wmv2/M3a1p2PyBJsFxN8xQMTEmTn3Wl/D26SsM7SpzbdW6gmfgoVU9Q==} - '@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@dbeda91': - resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@dbeda91} + '@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@69695cf': + resolution: {tarball: https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@69695cf} version: 1.0.0-next.85 peerDependencies: react-dom: ^18.0.0 @@ -2288,6 +2291,10 @@ packages: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} + ignore@6.0.2: + resolution: {integrity: sha512-InwqeHHN2XpumIkMvpl/DCJVrAHgCsG5+cn1XlnLWGwtZBm8QJfSusItfrwx81CTp5agNZqpKU2J/ccC5nGT4A==} + engines: {node: '>= 4'} + immutable@5.0.3: resolution: {integrity: sha512-P8IdPQHq3lA1xVeBRi5VPqUm5HDgKnx0Ru51wZz5mjxHr5n3RWhjIpOFU7ybkUxfB+5IToy+OLaHYDBIWsv+uw==} @@ -3989,15 +3996,15 @@ snapshots: dependencies: svelte: 4.2.19 - '@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@dbeda91e16325c2072088627718c0b637263cc76(svelte@4.2.19)': + '@appwrite.io/pink-icons-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@69695cf7bbcf4d49aa40ee2a2cfee6c46f6efac6(svelte@4.2.19)': dependencies: svelte: 4.2.19 '@appwrite.io/pink-icons@0.25.0': {} - '@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@dbeda91(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19)': + '@appwrite.io/pink-svelte@https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-svelte@69695cf(react-dom@18.3.1(react@18.3.1))(svelte@4.2.19)': dependencies: - '@appwrite.io/pink-icons-svelte': https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@dbeda91e16325c2072088627718c0b637263cc76(svelte@4.2.19) + '@appwrite.io/pink-icons-svelte': https://pkg.pr.new/appwrite/pink/@appwrite.io/pink-icons-svelte@69695cf7bbcf4d49aa40ee2a2cfee6c46f6efac6(svelte@4.2.19) '@floating-ui/dom': 1.6.12 '@melt-ui/svelte': 0.83.0(svelte@4.2.19) pretty-bytes: 6.1.1 @@ -6418,6 +6425,8 @@ snapshots: ignore@5.3.2: {} + ignore@6.0.2: {} + immutable@5.0.3: {} import-fresh@3.3.0: diff --git a/src/lib/components/uploadBox.svelte b/src/lib/components/uploadBox.svelte index 8cd9883cc..3fd4a89cd 100644 --- a/src/lib/components/uploadBox.svelte +++ b/src/lib/components/uploadBox.svelte @@ -45,7 +45,7 @@
diff --git a/src/lib/stores/uploader.ts b/src/lib/stores/uploader.ts index ff78d2523..fa463d9fc 100644 --- a/src/lib/stores/uploader.ts +++ b/src/lib/stores/uploader.ts @@ -1,11 +1,11 @@ -import { Client, type Models, Storage } from '@appwrite.io/console'; +import { Client, type Models, Sites, Storage } from '@appwrite.io/console'; import { writable } from 'svelte/store'; import { getProjectId } from '$lib/helpers/project'; import { getApiEndpoint } from '$lib/stores/sdk'; type UploaderFile = { $id: string; - bucketId: string; + resourceId: string; name: string; progress: number; completed: boolean; @@ -18,13 +18,16 @@ export type Uploader = { files: UploaderFile[]; }; -const temporaryStorage = () => { - const clientProject = new Client() - .setEndpoint(getApiEndpoint()) - .setMode('admin') - .setProject(getProjectId()); +const createAdminClient = () => { + return new Client().setEndpoint(getApiEndpoint()).setMode('admin').setProject(getProjectId()); +}; - return new Storage(clientProject); +const temporaryStorage = () => { + return new Storage(createAdminClient()); +}; + +const temporarySites = () => { + return new Sites(createAdminClient()); }; const createUploader = () => { @@ -69,7 +72,7 @@ const createUploader = () => { uploadFile: async (bucketId: string, id: string, file: File, permissions: string[]) => { const newFile: UploaderFile = { $id: id, - bucketId: bucketId, + resourceId: bucketId, name: file.name, progress: 0, completed: false, @@ -98,6 +101,40 @@ const createUploader = () => { newFile.completed = true; updateFile(newFile.$id, newFile); }, + uploadSiteDeployment: async (siteId: string, code: File) => { + const newDeployment: UploaderFile = { + $id: '', + resourceId: siteId, + name: code.name, + progress: 0, + completed: false, + failed: false + }; + update((n) => { + n.isOpen = true; + n.isCollapsed = false; + n.files.unshift(newDeployment); + return n; + }); + const uploadedFile = await temporarySites().createDeployment( + siteId, + code, + true, + undefined, + undefined, + undefined, + (p) => { + newDeployment.$id = p.$id; + newDeployment.progress = p.progress; + newDeployment.completed = p.progress === 100; + updateFile(p.$id, newDeployment); + } + ); + newDeployment.$id = uploadedFile.$id; + newDeployment.progress = 100; + newDeployment.completed = true; + updateFile(newDeployment.$id, newDeployment); + }, removeFromQueue: (id: string) => { update((n) => { n.files = n.files.filter((f) => f.$id !== id); diff --git a/src/routes/(console)/project-[project]/sites/site-[site]/deployments/createManualDeploymentModal.svelte b/src/routes/(console)/project-[project]/sites/site-[site]/deployments/createManualDeploymentModal.svelte index 86d84f43b..58b78903e 100644 --- a/src/routes/(console)/project-[project]/sites/site-[site]/deployments/createManualDeploymentModal.svelte +++ b/src/routes/(console)/project-[project]/sites/site-[site]/deployments/createManualDeploymentModal.svelte @@ -4,56 +4,100 @@ import { Dependencies } from '$lib/constants'; import { Button } from '$lib/elements/forms'; import { addNotification } from '$lib/stores/notifications'; - import { sdk } from '$lib/stores/sdk'; + import { uploader } from '$lib/stores/uploader'; import type { Models } from '@appwrite.io/console'; import { IconInfo } from '@appwrite.io/pink-icons-svelte'; import { Icon, Layout, Tooltip, Typography, Upload } from '@appwrite.io/pink-svelte'; - import { createTar, createTarGzip, type TarFileInput } from 'nanotar'; + import { createTarGzip } from 'nanotar'; + import { defaultIgnore } from './store'; + import ignore from 'ignore'; + interface FileData { + path: string; + buffer: ArrayBuffer; + } export let show = false; export let site: Models.Site; let files: FileList; - let uploadFile: File; - async function handleChange(event: CustomEvent) { - if (files?.length === 1) { - const file = files.item(0); - // If the file is a tar.gz file, then return - if (file.type === 'application/gzip' && file.name.split('.').pop() === 'tar') { - uploadFile = file; - } - } else { - let TarFileInputArray: TarFileInput[] = []; - Array.from(files).forEach((file) => { - let fileUint8Array: Uint8Array; - file.arrayBuffer().then((buff) => { - fileUint8Array = new Uint8Array(buff); - }); + let error: string = ''; - TarFileInputArray.push({ - name: file.name, - data: fileUint8Array - }); - }); - const tar = await createTarGzip(TarFileInputArray); - const blob = new Blob([tar], { type: 'application/gzip' }); - const file = new File([blob], 'deployment.tar.gz', { type: 'application/gzip' }); - uploadFile = file; + async function handleChange() { + const tick = performance.now(); + if (!files?.length) return; + + // If the file is a tar.gz file, then return it as is + if ( + files?.length === 1 && + files.item(0).type === 'application/gzip' && + files.item(0).name.split('.').pop() === 'tar' + ) { + uploadFile = files.item(0); } + // Else process the file to mantain the folder structure and create a .tar.gz file + else { + try { + const processedFiles = await processFileList(files); + const tar = await createTarGzip( + processedFiles.map((f) => ({ + name: f.path, + data: f.buffer + })) + ); + const blob = new Blob([tar], { type: 'application/gzip' }); + const file = new File([blob], 'deployment.tar.gz', { type: 'application/gzip' }); + uploadFile = file; + } catch (e) { + error = e; + } + } + console.log(uploadFile); + const tock = performance.now(); + console.log('Time taken to process files:', tock - tick); + } + + async function processFileList(files: FileList): Promise { + const fileArray = Array.from(files); + const ignorer = ignore(); + + // Check for .gitignore file + const gitignoreFile = fileArray.find( + (file) => file.webkitRelativePath.endsWith('.gitignore') || file.name === '.gitignore' + ); + + if (gitignoreFile) { + const gitignoreContent = await gitignoreFile.text(); + ignorer.add(gitignoreContent); + } else { + ignorer.add(defaultIgnore); + } + + // Filter files using ignorer + const filteredFiles = fileArray.filter((file) => { + const path = file.webkitRelativePath || file.name; + return !ignorer.ignores(path); + }); + const filePromises = filteredFiles.map(async (file) => { + try { + const buffer = await file.arrayBuffer(); + return { + path: file?.webkitRelativePath || file.name, + buffer: buffer + }; + } catch (e) { + console.error(error); + console.log(file); + error = e; + } + }); + + return Promise.all(filePromises); } async function createDeployment() { try { - await sdk.forProject.sites.createDeployment( - site.$id, - uploadFile, - true, //TODO: fix - undefined, - undefined, - undefined, - 'test' - ); + uploader.uploadSiteDeployment(site.$id, uploadFile); show = false; invalidate(Dependencies.DEPLOYMENTS); addNotification({ @@ -71,7 +115,7 @@ $: console.log(files); - + Manually deploy a site by uploading any file(s) or folder. @@ -89,8 +133,19 @@ Max file size: 10MB + {#if uploadFile} + + + {/if} - + diff --git a/src/routes/(console)/project-[project]/sites/site-[site]/deployments/store.ts b/src/routes/(console)/project-[project]/sites/site-[site]/deployments/store.ts index 70c47ae00..d22b3558a 100644 --- a/src/routes/(console)/project-[project]/sites/site-[site]/deployments/store.ts +++ b/src/routes/(console)/project-[project]/sites/site-[site]/deployments/store.ts @@ -100,3 +100,146 @@ export const columns = writable([ format: 'datetime' } ]); + +export const defaultIgnore = ` +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +### Node Patch ### +# Serverless Webpack directories +.webpack/ + +# Optional stylelint cache + +# SvelteKit build / generate output +.svelte-kit + +`;