diff --git a/package-lock.json b/package-lock.json index 87786c7..161912a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "helmet": "^7.1.0", "http-proxy-middleware": "^3.0.3", "jose": "^5.2.4", + "lru-cache": "^11.2.2", "openid-client": "^5.6.5", "prisma": "^5.13.0", "semver": "^7.6.3", @@ -31,6 +32,7 @@ "ws": "^8.17.1" }, "devDependencies": { + "@types/lru-cache": "^7.10.9", "@types/semver": "^7.5.8", "prettier": "3.2.5" }, @@ -1722,6 +1724,16 @@ "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz", "integrity": "sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==" }, + "node_modules/@types/lru-cache": { + "version": "7.10.9", + "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-7.10.9.tgz", + "integrity": "sha512-wrwgkdJ0xr8AbzKhVaRI8SXZN9saapPwwLoydBEr4HqMZET1LUTi1gdoaj82XmRJ9atqN7MtB0aja29iiK+7ag==", + "dev": true, + "license": "MIT", + "dependencies": { + "lru-cache": "*" + } + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -2526,14 +2538,12 @@ } }, "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", + "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", + "license": "ISC", "engines": { - "node": ">=10" + "node": "20 || >=22" } }, "node_modules/make-error": { @@ -2715,6 +2725,18 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/openid-client/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -3211,7 +3233,8 @@ "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" }, "node_modules/yn": { "version": "3.1.1", diff --git a/package.json b/package.json index 7ae23f7..73b0e73 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "helmet": "^7.1.0", "http-proxy-middleware": "^3.0.3", "jose": "^5.2.4", + "lru-cache": "^11.2.2", "openid-client": "^5.6.5", "prisma": "^5.13.0", "semver": "^7.6.3", @@ -44,7 +45,8 @@ "bufferutil": "^4.0.8" }, "devDependencies": { - "prettier": "3.2.5", - "@types/semver": "^7.5.8" + "@types/lru-cache": "^7.10.9", + "@types/semver": "^7.5.8", + "prettier": "3.2.5" } } diff --git a/src/helpers.ts b/src/helpers.ts new file mode 100644 index 0000000..4212bff --- /dev/null +++ b/src/helpers.ts @@ -0,0 +1,47 @@ + +import { createHash } from "crypto"; +import { type GetObjectCommandOutput } from "@aws-sdk/client-s3"; +import { InternalServerError } from "./errors"; +import { validRange } from "semver"; + +// Helper function to convert stream to string +export async function streamToString(stream: any): Promise { + const chunks: Uint8Array[] = []; + + for await (const chunk of stream) { + chunks.push(chunk); + } + + const result = Buffer.concat(chunks).toString("utf-8"); + return result.trimEnd(); +} + +// Helper function to convert stream to buffer +export async function streamToBuffer(stream: any): Promise { + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +export async function verifyHash( + file: GetObjectCommandOutput, + hashFile: GetObjectCommandOutput, + exception?: string, +): Promise { + const content = await streamToBuffer(file.Body); + const remoteHash = await streamToString(hashFile.Body); + const localHash = createHash("sha256").update(content).digest("hex"); + + const matches = remoteHash.trim() === localHash; + if (!matches && exception) { + throw new InternalServerError(exception); + } + return matches; +} + +export function toSemverRange(range?: string) { + if (!range) return "*"; + return validRange(range) || "*"; +} \ No newline at end of file diff --git a/src/releases.ts b/src/releases.ts index a7a0a3e..2db2bff 100644 --- a/src/releases.ts +++ b/src/releases.ts @@ -1,10 +1,21 @@ -import express from "express"; +import { Request, Response } from "express"; import { prisma } from "./db"; import { BadRequestError, InternalServerError, NotFoundError } from "./errors"; import { createHash } from "crypto"; import semver from "semver"; import { GetObjectCommand, ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3"; +import { LRUCache } from 'lru-cache'; + +import { streamToString, streamToBuffer, toSemverRange, verifyHash } from "./helpers"; + +export interface ReleaseMetadata { + version: string; + url: string; + hash: string; + _cachedAt?: number; + _maxSatisfying?: string; +} const s3Client = new S3Client({ endpoint: process.env.R2_ENDPOINT!, @@ -15,13 +26,30 @@ const s3Client = new S3Client({ region: "auto", }); +const releaseCache = new LRUCache({ + max: 1000, + ttl: 5 * 60 * 1000, // 5 minutes +}); + +const redirectCache = new LRUCache({ + max: 1000, + ttl: 5 * 60 * 1000, // 5 minutes +}); + const bucketName = process.env.R2_BUCKET; const baseUrl = process.env.R2_CDN_URL; async function getLatestVersion( prefix: "app" | "system", includePrerelease: boolean, -): Promise<{ version: string; url: string; hash: string }> { + maxSatisfying: string = "*", +): Promise { + const cacheKey = `${prefix}-${includePrerelease}-${maxSatisfying}`; + const cached = releaseCache.get(cacheKey); + if (cached) { + return cached; + } + const listCommand = new ListObjectsV2Command({ Bucket: bucketName, Prefix: prefix + "/", @@ -44,9 +72,12 @@ async function getLatestVersion( } // Get the latest version, optionally including prerelease versions - const latestVersion = semver.maxSatisfying(versions, "*", { + const latestVersion = semver.maxSatisfying(versions, maxSatisfying, { includePrerelease, }) as string; + if (!latestVersion) { + throw new NotFoundError(`No version found under prefix ${prefix} that satisfies ${maxSatisfying}`); + } const fileName = prefix === "app" ? "jetkvm_app" : "system.tar"; const url = `${baseUrl}/${prefix}/${latestVersion}/${fileName}`; @@ -59,32 +90,66 @@ async function getLatestVersion( ); const hash = await streamToString(hashResponse.Body); - return { version: latestVersion, url, hash }; + + // Cache the release metadata + const release = { + version: latestVersion, + url, + hash, + _cachedAt: Date.now(), + _maxSatisfying: maxSatisfying, + }; + releaseCache.set(cacheKey, release); + return release; } interface Release { appVersion: string; appUrl: string; appHash: string; + appCachedAt?: number; + appMaxSatisfying?: string; + systemVersion: string; systemUrl: string; systemHash: string; + systemCachedAt?: number; + systemMaxSatisfying?: string; } -async function getReleaseFromS3(includePrerelease: boolean): Promise { +function setAppRelease(release: Release, appRelease: ReleaseMetadata) { + release.appVersion = appRelease.version; + release.appUrl = appRelease.url; + release.appHash = appRelease.hash; + release.appCachedAt = appRelease._cachedAt; + release.appMaxSatisfying = appRelease._maxSatisfying; +} + +function setSystemRelease(release: Release, systemRelease: ReleaseMetadata) { + release.systemVersion = systemRelease.version; + release.systemUrl = systemRelease.url; + release.systemHash = systemRelease.hash; + release.systemCachedAt = systemRelease._cachedAt; + release.systemMaxSatisfying = systemRelease._maxSatisfying; +} + +function toRelease(appRelease?: ReleaseMetadata, systemRelease?: ReleaseMetadata): Release { + const release: Partial = {}; + if (appRelease) setAppRelease(release as Release, appRelease); + if (systemRelease) setSystemRelease(release as Release, systemRelease); + return release as Release; +} + +async function getReleaseFromS3( + includePrerelease: boolean, + { appVersion, systemVersion }: { appVersion?: string; systemVersion?: string }, +): Promise { const [appRelease, systemRelease] = await Promise.all([ - getLatestVersion("app", includePrerelease), - getLatestVersion("system", includePrerelease), + getLatestVersion("app", includePrerelease, appVersion), + getLatestVersion("system", includePrerelease, systemVersion), ]); - return { - appVersion: appRelease.version, - appUrl: appRelease.url, - appHash: appRelease.hash, - systemVersion: systemRelease.version, - systemUrl: systemRelease.url, - systemHash: systemRelease.hash, - }; + return toRelease(appRelease, systemRelease); } async function isDeviceEligibleForLatestRelease( @@ -126,7 +191,8 @@ async function getDefaultRelease(type: "app" | "system") { return latestDefaultRelease; } -export async function Retrieve(req: express.Request, res: express.Response) { +export async function Retrieve(req: Request, res: Response) { + // verify params const deviceId = req.query.deviceId as string | undefined; if (!deviceId) { throw new BadRequestError("Device ID is required"); @@ -134,27 +200,29 @@ export async function Retrieve(req: express.Request, res: express.Response) { const includePrerelease = req.query.prerelease === "true"; + const appVersion = toSemverRange(req.query.appVersion as string | undefined); + const systemVersion = toSemverRange(req.query.systemVersion as string | undefined); + const skipRollout = appVersion !== "*" || systemVersion !== "*"; + // Get the latest release from S3 let remoteRelease: Release; try { - remoteRelease = await getReleaseFromS3(includePrerelease); + remoteRelease = await getReleaseFromS3(includePrerelease, { appVersion, systemVersion }); } catch (error) { console.error(error); - throw new InternalServerError("Failed to get the latest release from S3"); + if (error instanceof NotFoundError) { + throw error; + } + throw new InternalServerError(`Failed to get the latest release from S3: ${error}`); } // If the request is for prereleases, ignore the rollout percentage and just return the latest release // This is useful for the OTA updater to get the latest prerelease version // This also prevents us from storing the rollout percentage for prerelease versions - if (includePrerelease) { - return res.json({ - appVersion: remoteRelease.appVersion, - appUrl: remoteRelease.appUrl, - appHash: remoteRelease.appHash, - systemVersion: remoteRelease.systemVersion, - systemUrl: remoteRelease.systemUrl, - systemHash: remoteRelease.systemHash, - }); + + // If the version isn't a wildcard, we skip the rollout percentage check + if (includePrerelease || skipRollout) { + return res.json(remoteRelease); } // Fetch or create the latest app release @@ -192,34 +260,20 @@ export async function Retrieve(req: express.Request, res: express.Response) { */ const forceUpdate = req.query.forceUpdate === "true"; if (forceUpdate) { - return res.json({ - appVersion: latestAppRelease.version, - appUrl: latestAppRelease.url, - appHash: latestAppRelease.hash, - systemVersion: latestSystemRelease.version, - systemUrl: latestSystemRelease.url, - systemHash: latestSystemRelease.hash, - }); + return res.json( + toRelease(latestAppRelease, latestSystemRelease), + ); } const defaultAppRelease = await getDefaultRelease("app"); const defaultSystemRelease = await getDefaultRelease("system"); - const responseJson = { - appVersion: defaultAppRelease.version, - appUrl: defaultAppRelease.url, - appHash: defaultAppRelease.hash, - systemVersion: defaultSystemRelease.version, - systemUrl: defaultSystemRelease.url, - systemHash: defaultSystemRelease.hash, - }; + const responseJson = toRelease(defaultAppRelease, defaultSystemRelease); if ( await isDeviceEligibleForLatestRelease(latestAppRelease.rolloutPercentage, deviceId) ) { - responseJson.appVersion = latestAppRelease.version; - responseJson.appUrl = latestAppRelease.url; - responseJson.appHash = latestAppRelease.hash; + setAppRelease(responseJson, latestAppRelease); } if ( @@ -228,148 +282,128 @@ export async function Retrieve(req: express.Request, res: express.Response) { deviceId, ) ) { - responseJson.systemVersion = latestSystemRelease.version; - responseJson.systemUrl = latestSystemRelease.url; - responseJson.systemHash = latestSystemRelease.hash; + setSystemRelease(responseJson, latestSystemRelease); } return res.json(responseJson); } -export async function RetrieveLatestSystemRecovery( - req: express.Request, - res: express.Response, -) { - const includePrerelease = req.query.prerelease === "true"; +function cachedRedirect(cachedKey: (req: Request) => string, callback: (req: Request) => Promise) { + return async (req: Request, res: Response) => { + const cacheKey = cachedKey(req); + let result = redirectCache.get(cacheKey); + if (!result) { + result = await callback(req); + redirectCache.set(cacheKey, result); + } + return res.redirect(302, result); + }; +} - // Get the latest system recovery image from S3. It's stored in the system/ folder. - const listCommand = new ListObjectsV2Command({ - Bucket: bucketName, - Prefix: "system/", - Delimiter: "/", - }); - const response = await s3Client.send(listCommand); +export const RetrieveLatestSystemRecovery = cachedRedirect( + (req: Request) => `system-recovery-${req.query.prerelease === "true" ? "pre" : "stable"}`, + async (req: Request) => { + const includePrerelease = req.query.prerelease === "true"; - // Extract version folder names - if (!response.CommonPrefixes || response.CommonPrefixes.length === 0) { - throw new NotFoundError(`No versions found under prefix system recovery image`); - } + // Get the latest system recovery image from S3. It's stored in the system/ folder. + const listCommand = new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: "system/", + Delimiter: "/", + }); + const response = await s3Client.send(listCommand); - // Get the latest version - const versions = response.CommonPrefixes.map(cp => cp.Prefix!.split("/")[1]) - .filter(Boolean) - .filter(v => semver.valid(v)); + // Extract version folder names + if (!response.CommonPrefixes || response.CommonPrefixes.length === 0) { + throw new NotFoundError(`No versions found under prefix system recovery image`); + } - const latestVersion = semver.maxSatisfying(versions, "*", { - includePrerelease, - }) as string; + // Get the latest version + const versions = response.CommonPrefixes.map(cp => cp.Prefix!.split("/")[1]) + .filter(Boolean) + .filter(v => semver.valid(v)); - const [firmwareFile, hashFile] = await Promise.all([ - s3Client.send( - new GetObjectCommand({ - Bucket: bucketName, - Key: `system/${latestVersion}/update.img`, - }), - ), - s3Client.send( - new GetObjectCommand({ - Bucket: bucketName, - Key: `system/${latestVersion}/update.img.sha256`, - }), - ), - ]); + const latestVersion = semver.maxSatisfying(versions, "*", { + includePrerelease, + }) as string; - if (!firmwareFile.Body || !hashFile.Body) { - throw new NotFoundError( - `No system recovery image or hash file not found for version ${latestVersion}`, + const [firmwareFile, hashFile] = await Promise.all([ + // TODO: store file hash using custom header to avoid extra request + s3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: `system/${latestVersion}/update.img`, + }), + ), + s3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: `system/${latestVersion}/update.img.sha256`, + }), + ), + ]); + + if (!firmwareFile.Body || !hashFile.Body) { + throw new NotFoundError( + `No system recovery image or hash file not found for version ${latestVersion}`, + ); + } + + await verifyHash(firmwareFile, hashFile, "system recovery image hash does not match"); + + console.log("system recovery image hash matches", latestVersion); + + return `${baseUrl}/system/${latestVersion}/update.img`; + }, +); + +export const RetrieveLatestApp = cachedRedirect( + (req: Request) => `app-${req.query.prerelease === "true" ? "pre" : "stable"}`, + async (req: Request) => { + const includePrerelease = req.query.prerelease === "true"; + + // Get the latest version + const listCommand = new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: "app/", + Delimiter: "/", + }); + const response = await s3Client.send(listCommand); + + if (!response.CommonPrefixes || response.CommonPrefixes.length === 0) { + throw new NotFoundError("No app versions found"); + } + + const versions = response.CommonPrefixes.map(cp => cp.Prefix!.split("/")[1]).filter(v => + semver.valid(v), ); - } - const firmwareContent = await streamToBuffer(firmwareFile.Body); - const remoteHash = await streamToString(hashFile.Body); - const localHash = createHash("sha256").update(firmwareContent).digest("hex"); + const latestVersion = semver.maxSatisfying(versions, "*", { + includePrerelease, + }) as string; - if (remoteHash.trim() !== localHash) { - throw new InternalServerError("system recovery image hash does not match"); - } + // Get the app file and its hash + const [appFile, hashFile] = await Promise.all([ + s3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: `app/${latestVersion}/jetkvm_app`, + }), + ), + s3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: `app/${latestVersion}/jetkvm_app.sha256`, + }), + ), + ]); - console.log("system recovery image hash matches", latestVersion); + if (!appFile.Body || !hashFile.Body) { + throw new NotFoundError(`App or hash file not found for version ${latestVersion}`); + } - return res.redirect(302, `${baseUrl}/system/${latestVersion}/update.img`); -} + await verifyHash(appFile, hashFile, "app hash does not match"); -export async function RetrieveLatestApp(req: express.Request, res: express.Response) { - const includePrerelease = req.query.prerelease === "true"; - - // Get the latest version - const listCommand = new ListObjectsV2Command({ - Bucket: bucketName, - Prefix: "app/", - Delimiter: "/", + console.log("App hash matches", latestVersion); + return `${baseUrl}/app/${latestVersion}/jetkvm_app`; }); - const response = await s3Client.send(listCommand); - - if (!response.CommonPrefixes || response.CommonPrefixes.length === 0) { - throw new NotFoundError("No app versions found"); - } - - const versions = response.CommonPrefixes.map(cp => cp.Prefix!.split("/")[1]).filter(v => - semver.valid(v), - ); - - const latestVersion = semver.maxSatisfying(versions, "*", { - includePrerelease, - }) as string; - - // Get the app file and its hash - const [appFile, hashFile] = await Promise.all([ - s3Client.send( - new GetObjectCommand({ - Bucket: bucketName, - Key: `app/${latestVersion}/jetkvm_app`, - }), - ), - s3Client.send( - new GetObjectCommand({ - Bucket: bucketName, - Key: `app/${latestVersion}/jetkvm_app.sha256`, - }), - ), - ]); - - if (!appFile.Body || !hashFile.Body) { - throw new NotFoundError(`App or hash file not found for version ${latestVersion}`); - } - - const appContent = await streamToBuffer(appFile.Body); - const remoteHash = await streamToString(hashFile.Body); - const localHash = createHash("sha256").update(appContent).digest("hex"); - - if (remoteHash.trim() !== localHash) { - throw new InternalServerError("App hash does not match"); - } - - console.log("App hash matches", latestVersion); - return res.redirect(302, `${baseUrl}/app/${latestVersion}/jetkvm_app`); -} - -// Helper function to convert stream to string -async function streamToString(stream: any): Promise { - const chunks: Uint8Array[] = []; - - for await (const chunk of stream) { - chunks.push(chunk); - } - - const result = Buffer.concat(chunks).toString("utf-8"); - return result.trimEnd(); -} - -// Helper function to convert stream to buffer -async function streamToBuffer(stream: any): Promise { - const chunks = []; - for await (const chunk of stream) { - chunks.push(chunk); - } - return Buffer.concat(chunks); -} diff --git a/tsconfig.json b/tsconfig.json index 1a064ac..5b26fe0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,7 @@ "extends": "@tsconfig/node22/tsconfig.json", "compilerOptions": { "baseUrl": ".", - "target": "es5", + "target": "es2022", "module": "NodeNext", "moduleResolution": "NodeNext", "outDir": "dist",