From 9da57a457b2378b8eb6afcd98df242fbc3edb70a Mon Sep 17 00:00:00 2001 From: hmacr Date: Mon, 21 Jul 2025 22:31:06 +0530 Subject: [PATCH 1/4] fix: Use direct source for file-preview when empty --- app/controllers/api/storage.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index c6e242296b..ef9cec1e73 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1072,6 +1072,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $decompressionTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime; + if (empty($source)) { + $source = $deviceForFiles->read($path); + } + try { $image = new Image($source); } catch (ImagickException $e) { @@ -1870,7 +1874,7 @@ App::get('/v1/storage/usage') $total = []; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $result = $dbForProject->findOne('stats', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -1899,7 +1903,7 @@ App::get('/v1/storage/usage') }; foreach ($metrics as $metric) { - $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['total'] = $stats[$metric]['total']; $usage[$metric]['data'] = []; $leap = time() - ($days['limit'] * $days['factor']); while ($leap < time()) { @@ -1917,8 +1921,8 @@ App::get('/v1/storage/usage') 'filesTotal' => $usage[$metrics[1]]['total'], 'filesStorageTotal' => $usage[$metrics[2]]['total'], 'buckets' => $usage[$metrics[0]]['data'], - 'files' => $usage[$metrics[1]]['data'], - 'storage' => $usage[$metrics[2]]['data'], + 'files' => $usage[$metrics[1]]['data'], + 'storage' => $usage[$metrics[2]]['data'], ]), Response::MODEL_USAGE_STORAGE); }); @@ -1970,7 +1974,7 @@ App::get('/v1/storage/:bucketId/usage') ? $dbForLogs : $dbForProject; - $result = $db->findOne('stats', [ + $result = $db->findOne('stats', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -2000,7 +2004,7 @@ App::get('/v1/storage/:bucketId/usage') }; foreach ($metrics as $metric) { - $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['total'] = $stats[$metric]['total']; $usage[$metric]['data'] = []; $leap = time() - ($days['limit'] * $days['factor']); while ($leap < time()) { From c29ef4087b03af60fb3516a11d791a1455e76914 Mon Sep 17 00:00:00 2001 From: hmacr Date: Wed, 23 Jul 2025 16:28:09 +0530 Subject: [PATCH 2/4] compress screenshots before uploading --- .../Platform/Modules/Functions/Workers/Builds.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 890c8572d9..86b536899c 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -35,6 +35,7 @@ use Utopia\Fetch\Client as FetchClient; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Compression\Algorithms\GZIP; use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -980,6 +981,9 @@ class Builds extends Action throw new \Exception($screenshotError); } + $mimeType = "image/png"; + $compressor = new GZIP(); + foreach ($screenshots as $data) { $key = $data['key']; $screenshot = $data['screenshot']; @@ -988,7 +992,7 @@ class Builds extends Action $fileName = $fileId . '.png'; $path = $deviceForFiles->getPath($fileName); $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root - $success = $deviceForFiles->write($path, $screenshot, "image/png"); + $success = $deviceForFiles->write($path, $compressor->compress($screenshot), $mimeType); if (!$success) { throw new \Exception("Screenshot failed to save"); @@ -1005,7 +1009,7 @@ class Builds extends Action 'name' => $fileName, 'path' => $path, 'signature' => $deviceForFiles->getFileHash($path), - 'mimeType' => $deviceForFiles->getFileMimeType($path), + 'mimeType' => $mimeType, 'sizeOriginal' => \strlen($screenshot), 'sizeActual' => $deviceForFiles->getFileSize($path), 'algorithm' => Compression::GZIP, @@ -1017,7 +1021,7 @@ class Builds extends Action 'openSSLTag' => null, 'openSSLIV' => null, 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => ['content_type' => $deviceForFiles->getFileMimeType($path)], + 'metadata' => ['content_type' => $mimeType], ]); Authorization::skip(fn () => $dbForPlatform->createDocument('bucket_' . $bucket->getSequence(), $file)); From b194558226d5588340338f5dc671b4f440992fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 23 Jul 2025 15:49:07 +0200 Subject: [PATCH 3/4] Switch from gzip to none compression --- app/controllers/api/storage.php | 4 ---- src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index ef9cec1e73..480f6c63c6 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1072,10 +1072,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $decompressionTime = \microtime(true) - $startTime - $downloadTime - $decryptionTime; - if (empty($source)) { - $source = $deviceForFiles->read($path); - } - try { $image = new Image($source); } catch (ImagickException $e) { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 86b536899c..82683d8d5f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -35,7 +35,6 @@ use Utopia\Fetch\Client as FetchClient; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; -use Utopia\Storage\Compression\Algorithms\GZIP; use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -982,7 +981,6 @@ class Builds extends Action } $mimeType = "image/png"; - $compressor = new GZIP(); foreach ($screenshots as $data) { $key = $data['key']; @@ -992,7 +990,7 @@ class Builds extends Action $fileName = $fileId . '.png'; $path = $deviceForFiles->getPath($fileName); $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root - $success = $deviceForFiles->write($path, $compressor->compress($screenshot), $mimeType); + $success = $deviceForFiles->write($path, $screenshot, $mimeType); if (!$success) { throw new \Exception("Screenshot failed to save"); @@ -1012,7 +1010,7 @@ class Builds extends Action 'mimeType' => $mimeType, 'sizeOriginal' => \strlen($screenshot), 'sizeActual' => $deviceForFiles->getFileSize($path), - 'algorithm' => Compression::GZIP, + 'algorithm' => Compression::NONE, 'comment' => '', 'chunksTotal' => 1, 'chunksUploaded' => 1, From 8240c9ccdc20298bab992b5043b3f931078669a1 Mon Sep 17 00:00:00 2001 From: hmacr Date: Wed, 23 Jul 2025 20:39:02 +0530 Subject: [PATCH 4/4] add tests --- .../Services/Sites/SitesConsoleClientTest.php | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/e2e/Services/Sites/SitesConsoleClientTest.php b/tests/e2e/Services/Sites/SitesConsoleClientTest.php index 28ce2a35ec..227e36a50e 100644 --- a/tests/e2e/Services/Sites/SitesConsoleClientTest.php +++ b/tests/e2e/Services/Sites/SitesConsoleClientTest.php @@ -92,12 +92,51 @@ class SitesConsoleClientTest extends Scope $this->assertNotEquals($screenshotDarkHash, $screenshotHash); + $screenshotId = $deployment['body']['screenshotLight']; $file = $this->client->call(Client::METHOD_GET, "/storage/buckets/screenshots/files/$screenshotId/view?project=console"); $this->assertEquals(404, $file['headers']['status-code']); + $screenshotId = $deployment['body']['screenshotDark']; $file = $this->client->call(Client::METHOD_GET, "/storage/buckets/screenshots/files/$screenshotId/view?project=console"); $this->assertEquals(404, $file['headers']['status-code']); + // Verify previews + $screenshotId = $deployment['body']['screenshotLight']; + $file = $this->client->call(Client::METHOD_GET, "/storage/buckets/screenshots/files/$screenshotId/preview?project=console", array_merge($this->getHeaders(), [ + 'x-appwrite-mode' => 'default' // NOT ADMIN! + ])); + + $this->assertEquals(200, $file['headers']['status-code']); + $this->assertNotEmpty(200, $file['body']); + $this->assertGreaterThan(1, $file['headers']['content-length']); + $this->assertEquals('image/png', $file['headers']['content-type']); + + $screenshotHash = \md5($file['body']); + $this->assertNotEmpty($screenshotHash); + + $screenshotId = $deployment['body']['screenshotDark']; + $file = $this->client->call(Client::METHOD_GET, "/storage/buckets/screenshots/files/$screenshotId/preview?project=console", array_merge($this->getHeaders(), [ + 'x-appwrite-mode' => 'default' // NOT ADMIN! + ])); + + $this->assertEquals(200, $file['headers']['status-code']); + $this->assertNotEmpty(200, $file['body']); + $this->assertGreaterThan(1, $file['headers']['content-length']); + $this->assertEquals('image/png', $file['headers']['content-type']); + + $screenshotDarkHash = \md5($file['body']); + $this->assertNotEmpty($screenshotDarkHash); + + $this->assertNotEquals($screenshotDarkHash, $screenshotHash); + + $screenshotId = $deployment['body']['screenshotLight']; + $file = $this->client->call(Client::METHOD_GET, "/storage/buckets/screenshots/files/$screenshotId/preview?project=console"); + $this->assertEquals(404, $file['headers']['status-code']); + + $screenshotId = $deployment['body']['screenshotDark']; + $file = $this->client->call(Client::METHOD_GET, "/storage/buckets/screenshots/files/$screenshotId/preview?project=console"); + $this->assertEquals(404, $file['headers']['status-code']); + $this->cleanupSite($siteId); } }